From 1a863c3ecfa04eacf7df3c5bce71c12244e4625b Mon Sep 17 00:00:00 2001 From: Daniel Karbach Date: Fri, 10 May 2024 18:34:34 +0200 Subject: [PATCH 01/14] universal examples --- app/Models/ChatLib.php | 92 +++++++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 42 deletions(-) diff --git a/app/Models/ChatLib.php b/app/Models/ChatLib.php index a87c6a7..c7a19b6 100644 --- a/app/Models/ChatLib.php +++ b/app/Models/ChatLib.php @@ -6,7 +6,7 @@ use Illuminate\Support\Facades\Storage; class ChatLib { - public function __construct($size = 7) { + public function __construct($size = 6) { $this->size = $size; $converted = []; @@ -39,16 +39,27 @@ class ChatLib { if ($end - $i < 5) break; } } + $this->addExample(array_slice($tokens, 0, $num), $token); } } public function compile() { - foreach ($this->transitions as $key => $value) { - $this->transitions[$key] = $this->index($this->transitions[$key]); + foreach ($this->transitions as $key => $values) { + $this->transitions[$key] = $this->index($values, 2); if (empty($this->transitions[$key])) { unset($this->transitions[$key]); } } + foreach ($this->examples as $key => $values) { + if (in_array($key, ['', ' '])) { + unset($this->examples[$key]); + continue; + } + $this->examples[$key] = $this->index($values, 1); + if (empty($this->examples[$key]) || (count($this->examples[$key]) === 1 && $this->examples[$key][0][0] === $key)) { + unset($this->examples[$key]); + } + } } public function generate($limit = 100) { @@ -67,6 +78,7 @@ class ChatLib { $data = [ 'size' => $this->size, 'transitions' => $this->transitions, + 'examples' => $this->examples, ]; Storage::disk('chatlib')->put($name.'.json', json_encode($data)); } @@ -75,28 +87,17 @@ class ChatLib { $data = json_decode(Storage::disk('chatlib')->get($name.'.json'), true); $this->size = $data['size']; $this->transitions = $data['transitions']; + $this->examples = $data['examples']; } - private function index($arr) { + private function index($arr, $min_weight = 2) { $result = []; $sum = 0; - foreach ($arr as $key => $entry) { - $weight = $entry[0]; - if ($weight == 1) continue; + foreach ($arr as $key => $weight) { + if ($weight < $min_weight) continue; $lower = $sum; $sum += $weight; - $examples = []; - if ($key === ' ') { - $examples = [[' ', 0, 1]]; - } else { - $subsum = 0; - foreach ($entry[1] as $example => $subweight) { - $sublower = $subsum; - $subsum += $subweight; - $examples[] = [$example, $sublower, $subsum]; - } - } - $result[] = [$key, $lower, $sum, $examples]; + $result[] = [$key, $lower, $sum]; } return $result; } @@ -108,7 +109,7 @@ class ChatLib { if (isset($this->transitions[$cmb])) { $pick = $this->pick($this->transitions[$cmb]); if (!is_null($pick)) { - return $this->exampleOf($pick); + return $this->exampleOf($pick, $tokens); } } } @@ -142,28 +143,27 @@ class ChatLib { } private function addTransition($state, $next) { - $cmb = $this->generalize($state); - if (!isset($this->transitions[$cmb])) { - $this->transitions[$cmb] = []; + $ctx = $this->generalize($state); + $cmb = $this->generalize([$next]); + if (!isset($this->transitions[$ctx])) { + $this->transitions[$ctx] = []; + } + if (!isset($this->transitions[$ctx][$cmb])) { + $this->transitions[$ctx][$cmb] = 1; + } else { + ++$this->transitions[$ctx][$cmb]; } - $this->increment($this->transitions[$cmb], $next); } - private function increment(&$which, $token) { - $generalized = $this->generalize([$token]); - if (!isset($which[$generalized])) { - $which[$generalized] = [ - 1, - [], - ]; - $which[$generalized][1][$token] = 1; + private function addExample($context, $token) { + $cmb = $this->generalize([$token]); + if (!isset($this->examples[$cmb])) { + $this->examples[$cmb] = []; + } + if (!isset($this->examples[$cmb][$token])) { + $this->examples[$cmb][$token] = 1; } else { - ++$which[$generalized][0]; - if (!isset($which[$generalized][1][$token])) { - $which[$generalized][1][$token] = 1; - } else { - ++$which[$generalized][1][$token]; - } + ++$this->examples[$cmb][$token]; } } @@ -199,13 +199,20 @@ class ChatLib { return $str; } - private function exampleOf($pick) { - $example = $this->pick($pick[3]); - return $example[0]; + private function exampleOf($pick, $context) { + if (!isset($this->examples[$pick[0]])) { + return $pick[0]; + } + if (isset($this->examples[$pick[0]])) { + $example = $this->pick($this->examples[$pick[0]]); + return $example[0]; + } + return $pick[0]; } - private $size = 7; + private $size; private $transitions = []; + private $examples = []; private $aliases = [ 'chest' => ['kiste'], @@ -358,6 +365,7 @@ class ChatLib { 'wave' => [ 'dennsenhi', 'dergoawave', + 'falcnwavehi', 'heyguys', 'holysm0heyguys', 'muftaahey', -- 2.39.2 From c96f2eb4ae81141c04241c673525569fef9cbaeb Mon Sep 17 00:00:00 2001 From: Daniel Karbach Date: Tue, 21 May 2024 11:13:45 +0200 Subject: [PATCH 02/14] fix editorconfig --- .editorconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.editorconfig b/.editorconfig index 1671c9b..49dd9a0 100644 --- a/.editorconfig +++ b/.editorconfig @@ -4,7 +4,7 @@ root = true charset = utf-8 end_of_line = lf insert_final_newline = true -indent_style = space +indent_style = tab indent_size = 4 trim_trailing_whitespace = true -- 2.39.2 From ff8fc77e4fd1686bc1c1b34d094b11e6dfd93516 Mon Sep 17 00:00:00 2001 From: Daniel Karbach Date: Tue, 21 May 2024 11:18:38 +0200 Subject: [PATCH 03/14] fix zsr sync --- app/Console/Commands/SyncZSR.php | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/app/Console/Commands/SyncZSR.php b/app/Console/Commands/SyncZSR.php index 1d92110..65b21c1 100644 --- a/app/Console/Commands/SyncZSR.php +++ b/app/Console/Commands/SyncZSR.php @@ -88,17 +88,6 @@ class SyncZSR extends Command { } else { $episode->title = $entry['summary']; } - if (preg_match('/Restream: https?:\/\/(www\.)?twitch\.tv\/(\w+)/u', $entry['description'], $matches)) { - $channel = $this->syncChannel($episode, $matches[2]); - if ($channel) { - $episode->channels()->syncWithoutDetaching([$channel->id]); - } - } - if (preg_match('/^(.*) - (.*?) vs (.*?)$/u', $episode->title, $matches)) { - $episode->title = $matches[1]; - $this->syncPlayer($episode, $matches[2]); - $this->syncPlayer($episode, $matches[3]); - } $start = Carbon::parse($entry['start']['dateTime'])->setTimezone('UTC'); if (!$episode->start || $start->ne($episode->start)) { $episode->start = $start; @@ -106,7 +95,20 @@ class SyncZSR extends Command { $end = Carbon::parse($entry['end']['dateTime'])->setTimezone('UTC'); $episode->estimate = $start->diffInSeconds($end); $episode->confirmed = true; - $episode->save(); + if (preg_match('/^(.*) - (.*?) vs\.? (.*?)$/u', $episode->title, $matches)) { + $episode->title = $matches[1]; + $episode->save(); + $this->syncPlayer($episode, $matches[2]); + $this->syncPlayer($episode, $matches[3]); + } else { + $episode->save(); + } + if (preg_match('/Restream: https?:\/\/(www\.)?twitch\.tv\/(\w+)/u', $entry['description'], $matches)) { + $channel = $this->syncChannel($episode, $matches[2]); + if ($channel) { + $episode->channels()->syncWithoutDetaching([$channel->id]); + } + } } private function syncChannel(Episode $episode, $zsrChannel) { -- 2.39.2 From 7aa9c88df0a0ac66c4a2f8473705aed8ccc65d06 Mon Sep 17 00:00:00 2001 From: Daniel Karbach Date: Tue, 21 May 2024 11:41:17 +0200 Subject: [PATCH 04/14] auto sync zsr every 15 minutes --- app/Console/Kernel.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 32e5c8a..14c7840 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -19,6 +19,7 @@ class Kernel extends ConsoleKernel $schedule->command('sync:ladder')->daily(); $schedule->command('sync:speedgaming')->everyFiveMinutes(); $schedule->command('sync:sra')->everyFifteenMinutes(); + $schedule->command('sync:zsr')->everyFifteenMinutes(); $schedule->command('sync:avatars')->everyFiveMinutes(); $schedule->command('sync:racetime')->everyFiveMinutes(); $schedule->command('chat:evaluate 100')->everyMinute(); -- 2.39.2 From bda7ebfd0ba79a30d4e02b321ed4abdb89840f6e Mon Sep 17 00:00:00 2001 From: Daniel Karbach Date: Thu, 23 May 2024 16:24:55 +0200 Subject: [PATCH 05/14] blacklist known spam --- app/TwitchBot/TokenizedMessage.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/TwitchBot/TokenizedMessage.php b/app/TwitchBot/TokenizedMessage.php index 4b7d758..7d7438a 100644 --- a/app/TwitchBot/TokenizedMessage.php +++ b/app/TwitchBot/TokenizedMessage.php @@ -259,7 +259,14 @@ class TokenizedMessage { if ($this->containsRaw('horsti')) { return true; } - if ($this->containsRaw(['folgtjetzt', 'vielendankfürdenraid', 'thanksfortheraid', 'willkommenaufstarbase47'])) { + if ($this->containsRaw([ + 'folgtjetzt', + 'hatdeinenkanalgeraided', + 'stürmtdenladenmit', + 'thanksfortheraid', + 'vielendankfürdenraid', + 'willkommenaufstarbase47', + ])) { return true; } return false; -- 2.39.2 From 9a606c41e6ae92d82dbace81a28b4e54ab10c9cb Mon Sep 17 00:00:00 2001 From: Daniel Karbach Date: Thu, 23 May 2024 21:14:12 +0200 Subject: [PATCH 06/14] more spam shit --- app/TwitchBot/TokenizedMessage.php | 1 + tests/Unit/TwitchBot/TokenizedMessageTest.php | 1 + 2 files changed, 2 insertions(+) diff --git a/app/TwitchBot/TokenizedMessage.php b/app/TwitchBot/TokenizedMessage.php index 7d7438a..f84527c 100644 --- a/app/TwitchBot/TokenizedMessage.php +++ b/app/TwitchBot/TokenizedMessage.php @@ -262,6 +262,7 @@ class TokenizedMessage { if ($this->containsRaw([ 'folgtjetzt', 'hatdeinenkanalgeraided', + 'isnowlivestreaming', 'stürmtdenladenmit', 'thanksfortheraid', 'vielendankfürdenraid', diff --git a/tests/Unit/TwitchBot/TokenizedMessageTest.php b/tests/Unit/TwitchBot/TokenizedMessageTest.php index 832b970..1380417 100644 --- a/tests/Unit/TwitchBot/TokenizedMessageTest.php +++ b/tests/Unit/TwitchBot/TokenizedMessageTest.php @@ -88,6 +88,7 @@ class TokenizedMessageTest extends TestCase { $this->assertTrue(TokenizedMessage::fromString('hello would you like some followers?')->isSpammy()); $this->assertTrue(TokenizedMessage::fromString('get view ers for free')->isSpammy()); + $this->assertTrue(TokenizedMessage::fromString('XallGG is now live! Streaming The Legend of Zelda: A Link to the Past: Casual Boots Seed zum Spaß/Practice')->isSpammy()); $this->assertTrue(TokenizedMessage::fromString('also bitte, horstie')->isSpammy()); $this->assertTrue(TokenizedMessage::fromString('hey maengi, vielen dank für den raid')->isSpammy()); -- 2.39.2 From d650f944d661debb64954ac590d978b07619a431 Mon Sep 17 00:00:00 2001 From: Daniel Karbach Date: Tue, 28 May 2024 09:41:37 +0200 Subject: [PATCH 07/14] fix model select blur behaviour --- resources/js/components/common/ChannelSelect.js | 4 ++-- resources/js/components/common/DiscordChannelSelect.js | 6 ++---- resources/js/components/common/DiscordSelect.js | 4 ++-- resources/js/components/common/UserSelect.js | 4 ++-- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/resources/js/components/common/ChannelSelect.js b/resources/js/components/common/ChannelSelect.js index a69662f..355b02b 100644 --- a/resources/js/components/common/ChannelSelect.js +++ b/resources/js/components/common/ChannelSelect.js @@ -29,10 +29,10 @@ const ChannelSelect = ({ setShowResults(false); } }; - document.addEventListener('click', handleEventOutside, true); + document.addEventListener('mousedown', handleEventOutside, true); document.addEventListener('focus', handleEventOutside, true); return () => { - document.removeEventListener('click', handleEventOutside, true); + document.removeEventListener('mousedown', handleEventOutside, true); document.removeEventListener('focus', handleEventOutside, true); }; }, []); diff --git a/resources/js/components/common/DiscordChannelSelect.js b/resources/js/components/common/DiscordChannelSelect.js index fd5fe8d..01ee3b0 100644 --- a/resources/js/components/common/DiscordChannelSelect.js +++ b/resources/js/components/common/DiscordChannelSelect.js @@ -10,9 +10,7 @@ import debounce from '../../helpers/debounce'; const DiscordChannelSelect = ({ guild, - isInvalid, name, - onBlur, onChange, types, value, @@ -31,10 +29,10 @@ const DiscordChannelSelect = ({ setShowResults(false); } }; - document.addEventListener('click', handleEventOutside, true); + document.addEventListener('mousedown', handleEventOutside, true); document.addEventListener('focus', handleEventOutside, true); return () => { - document.removeEventListener('click', handleEventOutside, true); + document.removeEventListener('mousedown', handleEventOutside, true); document.removeEventListener('focus', handleEventOutside, true); }; }, []); diff --git a/resources/js/components/common/DiscordSelect.js b/resources/js/components/common/DiscordSelect.js index 65c94d6..9ac7b56 100644 --- a/resources/js/components/common/DiscordSelect.js +++ b/resources/js/components/common/DiscordSelect.js @@ -23,10 +23,10 @@ const DiscordSelect = ({ onChange, value }) => { setShowResults(false); } }; - document.addEventListener('click', handleEventOutside, true); + document.addEventListener('mousedown', handleEventOutside, true); document.addEventListener('focus', handleEventOutside, true); return () => { - document.removeEventListener('click', handleEventOutside, true); + document.removeEventListener('mousedown', handleEventOutside, true); document.removeEventListener('focus', handleEventOutside, true); }; }, []); diff --git a/resources/js/components/common/UserSelect.js b/resources/js/components/common/UserSelect.js index 03aa097..32bd186 100644 --- a/resources/js/components/common/UserSelect.js +++ b/resources/js/components/common/UserSelect.js @@ -21,10 +21,10 @@ const UserSelect = ({ name, onChange, value }) => { setShowResults(false); } }; - document.addEventListener('click', handleEventOutside, true); + document.addEventListener('mousedown', handleEventOutside, true); document.addEventListener('focus', handleEventOutside, true); return () => { - document.removeEventListener('click', handleEventOutside, true); + document.removeEventListener('mousedown', handleEventOutside, true); document.removeEventListener('focus', handleEventOutside, true); }; }, []); -- 2.39.2 From ee18df826f15227777b390f4fceabd0067575430 Mon Sep 17 00:00:00 2001 From: Daniel Karbach Date: Wed, 29 May 2024 11:13:32 +0200 Subject: [PATCH 08/14] use slug for SG twitch channels this should fix SGFR --- app/Console/Commands/SyncSpeedGaming.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Console/Commands/SyncSpeedGaming.php b/app/Console/Commands/SyncSpeedGaming.php index 41286ce..1b4c80e 100644 --- a/app/Console/Commands/SyncSpeedGaming.php +++ b/app/Console/Commands/SyncSpeedGaming.php @@ -207,7 +207,7 @@ class SyncSpeedGaming extends Command { } $channel->short_name = $sgChannel['initials']; $channel->title = $sgChannel['name']; - $channel->stream_link = 'https://twitch.tv/'.strtolower($sgChannel['name']); + $channel->stream_link = 'https://twitch.tv/'.strtolower($sgChannel['slug']); $channel->languages = [$sgChannel['language']]; $channel->save(); return $channel; -- 2.39.2 From 911c3644cd3a7203bf39e84b686d8fbeb07d9fa1 Mon Sep 17 00:00:00 2001 From: Daniel Karbach Date: Mon, 3 Jun 2024 15:38:56 +0200 Subject: [PATCH 09/14] more spam --- app/TwitchBot/TokenizedMessage.php | 1 + tests/Unit/TwitchBot/TokenizedMessageTest.php | 1 + 2 files changed, 2 insertions(+) diff --git a/app/TwitchBot/TokenizedMessage.php b/app/TwitchBot/TokenizedMessage.php index f84527c..b460020 100644 --- a/app/TwitchBot/TokenizedMessage.php +++ b/app/TwitchBot/TokenizedMessage.php @@ -265,6 +265,7 @@ class TokenizedMessage { 'isnowlivestreaming', 'stürmtdenladenmit', 'thanksfortheraid', + 'verschwindetfürneweileindenlurk', 'vielendankfürdenraid', 'willkommenaufstarbase47', ])) { diff --git a/tests/Unit/TwitchBot/TokenizedMessageTest.php b/tests/Unit/TwitchBot/TokenizedMessageTest.php index 1380417..be5da8d 100644 --- a/tests/Unit/TwitchBot/TokenizedMessageTest.php +++ b/tests/Unit/TwitchBot/TokenizedMessageTest.php @@ -88,6 +88,7 @@ class TokenizedMessageTest extends TestCase { $this->assertTrue(TokenizedMessage::fromString('hello would you like some followers?')->isSpammy()); $this->assertTrue(TokenizedMessage::fromString('get view ers for free')->isSpammy()); + $this->assertTrue(TokenizedMessage::fromString('yayklaygaming verschwindet für \'ne Weile in den Lurk. Cool, dass Du vorbeigeschaut hast xallggCheers PogChamp')->isSpammy()); $this->assertTrue(TokenizedMessage::fromString('XallGG is now live! Streaming The Legend of Zelda: A Link to the Past: Casual Boots Seed zum Spaß/Practice')->isSpammy()); $this->assertTrue(TokenizedMessage::fromString('also bitte, horstie')->isSpammy()); -- 2.39.2 From 5df7f3ed455c56370b3c4b690b0ca47b183fa829 Mon Sep 17 00:00:00 2001 From: Daniel Karbach Date: Sat, 8 Jun 2024 00:13:13 +0200 Subject: [PATCH 10/14] add up to 4 player support for zsr sync --- app/Console/Commands/SyncZSR.php | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/app/Console/Commands/SyncZSR.php b/app/Console/Commands/SyncZSR.php index 65b21c1..b7719e2 100644 --- a/app/Console/Commands/SyncZSR.php +++ b/app/Console/Commands/SyncZSR.php @@ -95,7 +95,20 @@ class SyncZSR extends Command { $end = Carbon::parse($entry['end']['dateTime'])->setTimezone('UTC'); $episode->estimate = $start->diffInSeconds($end); $episode->confirmed = true; - if (preg_match('/^(.*) - (.*?) vs\.? (.*?)$/u', $episode->title, $matches)) { + if (preg_match('/^(.*) - (.*?) vs\.? (.*?) vs\.? (.*?) vs\.? (.*?)$/u', $episode->title, $matches)) { + $episode->title = $matches[1]; + $episode->save(); + $this->syncPlayer($episode, $matches[2]); + $this->syncPlayer($episode, $matches[3]); + $this->syncPlayer($episode, $matches[4]); + $this->syncPlayer($episode, $matches[5]); + } else if (preg_match('/^(.*) - (.*?) vs\.? (.*?) vs\.? (.*?)$/u', $episode->title, $matches)) { + $episode->title = $matches[1]; + $episode->save(); + $this->syncPlayer($episode, $matches[2]); + $this->syncPlayer($episode, $matches[3]); + $this->syncPlayer($episode, $matches[4]); + } else if (preg_match('/^(.*) - (.*?) vs\.? (.*?)$/u', $episode->title, $matches)) { $episode->title = $matches[1]; $episode->save(); $this->syncPlayer($episode, $matches[2]); -- 2.39.2 From 1dea58cb6fa9cf28966e75c1e1af87f67e6c0fd1 Mon Sep 17 00:00:00 2001 From: Daniel Karbach Date: Thu, 27 Jun 2024 19:01:36 +0200 Subject: [PATCH 11/14] revamp chatlib tokenization --- app/Console/Commands/ChatlibDatabase.php | 9 +- app/Console/Commands/ChatlibGenerate.php | 4 +- .../Commands/ReevaluateChatCommand.php | 8 +- app/Models/ChatLib.php | 198 +++++++----------- app/Models/ChatLog.php | 27 ++- app/TwitchBot/TokenizedMessage.php | 7 +- 6 files changed, 124 insertions(+), 129 deletions(-) diff --git a/app/Console/Commands/ChatlibDatabase.php b/app/Console/Commands/ChatlibDatabase.php index e492bb7..9bf143f 100644 --- a/app/Console/Commands/ChatlibDatabase.php +++ b/app/Console/Commands/ChatlibDatabase.php @@ -13,7 +13,7 @@ class ChatlibDatabase extends Command { * * @var string */ - protected $signature = 'chatlib:database {which=de} {size=7}'; + protected $signature = 'chatlib:database {which=de} {size=3}'; /** * The console command description. @@ -44,10 +44,13 @@ class ChatlibDatabase extends Command { $query->whereNull('detected_language'); $query->orWhere('detected_language', '=', $lang); }) - ->whereRaw('LENGTH(`text_content`) > 10') + ->orderBy('channel_id') + ->orderBy('created_at') ->chunk(5000, function ($msgs) use (&$count, $db) { + $previous = null; foreach ($msgs as $msg) { - $db->addMessage($msg); + $db->addMessage($msg, $previous); + $previous = $msg; ++$count; } $this->line($count); diff --git a/app/Console/Commands/ChatlibGenerate.php b/app/Console/Commands/ChatlibGenerate.php index 04c9e41..caa223d 100644 --- a/app/Console/Commands/ChatlibGenerate.php +++ b/app/Console/Commands/ChatlibGenerate.php @@ -12,7 +12,7 @@ class ChatlibGenerate extends Command { * * @var string */ - protected $signature = 'chatlib:generate {which=de} {amount=50}'; + protected $signature = 'chatlib:generate {which=de} {amount=50} {context?}'; /** * The console command description. @@ -39,7 +39,7 @@ class ChatlibGenerate extends Command { $amount = intval($this->argument('amount')); for ($i = 0; $i < $amount; ++$i) { - $this->line($db->generate()); + $this->line($db->generate($this->argument('context'))); } return 0; diff --git a/app/Console/Commands/ReevaluateChatCommand.php b/app/Console/Commands/ReevaluateChatCommand.php index b289bd0..d87179c 100644 --- a/app/Console/Commands/ReevaluateChatCommand.php +++ b/app/Console/Commands/ReevaluateChatCommand.php @@ -32,12 +32,14 @@ class ReevaluateChatCommand extends Command { ChatLog::whereIn('type', ['chat', 'error']) ->where('banned', false) ->orderBy('created_at') - ->chunk(10000, function ($logs) use (&$good, &$bad) { + ->chunk(5000, function ($logs) use (&$good, &$bad) { foreach ($logs as $line) { try { $line->evaluate(); - $line->evaluated_at = now(); - $line->save(); + if ($line->isDirty()) { + $line->evaluated_at = now(); + $line->save(); + } ++$good; } catch (\Exception $e) { ++$bad; diff --git a/app/Models/ChatLib.php b/app/Models/ChatLib.php index c7a19b6..d6a7ffc 100644 --- a/app/Models/ChatLib.php +++ b/app/Models/ChatLib.php @@ -6,79 +6,65 @@ use Illuminate\Support\Facades\Storage; class ChatLib { - public function __construct($size = 6) { + public function __construct($size = 3) { $this->size = $size; - $converted = []; foreach ($this->categories as $category => $patterns) { $converted_patterns = []; foreach ($patterns as $pattern) { $converted_patterns[] = '/\b'.$pattern.'\b/u'; } - $converted['%'.strtoupper($category).'%'] = $converted_patterns; + $converted[strtoupper($category)] = $converted_patterns; } $this->categories = $converted; } - public function addMessage(ChatLog $msg) { - $this->addText($msg->text_content); + public function addMessage(ChatLog $msg, ChatLog $previous = null) { + if ($msg->isReply()) { + $this->addText($msg->text_content, $msg->getReplyParent()); + } else if (!is_null($previous)) { + $this->addText($msg->text_content, $previous->text_content); + } else { + $this->addText($msg->text_content); + } } - public function addText($text) { + public function addText($text, $context = '') { $tokens = $this->tokenize($text); - if (empty($tokens)) return; - $tokens[] = ''; - foreach ($tokens as $num => $token) { - if ($num === 0) { - $this->addTransition([], $token); - } else { - $start = max(0, $num - $this->size - 1); - $end = $num; - for ($i = $start; $i < $end; ++$i) { - $this->addTransition(array_slice($tokens, $i, $end - $i), $token); - if ($end - $i < 5) break; - } + for ($i = 0; $i < count($tokens) - $this->size; ++$i) { + $this->addTransition(array_slice($tokens, $i, $this->size), $tokens[$i + $this->size]); + } + if (!empty($context)) { + $tokens = $this->tokenizeWithContext($text, $context); + $size = min($this->size - 1, count($tokens) - $this->size); + for ($i = 0; $i < $size; ++$i) { + $this->addTransition(array_slice($tokens, $i, $this->size), $tokens[$i + $this->size]); } - $this->addExample(array_slice($tokens, 0, $num), $token); } } public function compile() { foreach ($this->transitions as $key => $values) { - $this->transitions[$key] = $this->index($values, 2); - if (empty($this->transitions[$key])) { - unset($this->transitions[$key]); - } - } - foreach ($this->examples as $key => $values) { - if (in_array($key, ['', ' '])) { - unset($this->examples[$key]); - continue; - } - $this->examples[$key] = $this->index($values, 1); - if (empty($this->examples[$key]) || (count($this->examples[$key]) === 1 && $this->examples[$key][0][0] === $key)) { - unset($this->examples[$key]); - } + $this->transitions[$key] = $this->index($values); } } - public function generate($limit = 100) { - $tokens = ['']; - $generated = ''; - while (strlen($generated) < $limit) { - $next = $this->randomNext($tokens); - if ($next === '') break; - $tokens[] = $next; - $generated .= $next; + public function generate($context = null) { + if (!is_null($context)) { + $tokens = $this->tokenizeWithContext('', $context); + $generated = $this->loop($tokens); + if (!empty($generated)) { + return $generated; + } } - return $generated; + $tokens = $this->tokenize(''); + return $this->loop($tokens); } public function saveAs($name) { $data = [ 'size' => $this->size, 'transitions' => $this->transitions, - 'examples' => $this->examples, ]; Storage::disk('chatlib')->put($name.'.json', json_encode($data)); } @@ -87,14 +73,12 @@ class ChatLib { $data = json_decode(Storage::disk('chatlib')->get($name.'.json'), true); $this->size = $data['size']; $this->transitions = $data['transitions']; - $this->examples = $data['examples']; } - private function index($arr, $min_weight = 2) { + private function index($arr) { $result = []; $sum = 0; foreach ($arr as $key => $weight) { - if ($weight < $min_weight) continue; $lower = $sum; $sum += $weight; $result[] = [$key, $lower, $sum]; @@ -102,18 +86,20 @@ class ChatLib { return $result; } - private function randomNext($tokens) { - $cnt = count($tokens); - for ($size = min($this->size, $cnt); $size > 0; --$size) { - $cmb = $this->generalize(array_slice($tokens, -$size)); - if (isset($this->transitions[$cmb])) { - $pick = $this->pick($this->transitions[$cmb]); - if (!is_null($pick)) { - return $this->exampleOf($pick, $tokens); - } - } + private function loop($tokens) { + while (count($tokens) < 50) { + $next = $this->randomNext($tokens); + if ($next === ' ') break; + $tokens[] = $next; } - return ''; + return $this->untokenize($tokens); + } + + private function randomNext($tokens) { + $key = $this->makeKey($tokens); + if (!isset($this->transitions[$key])) return ' '; + $pick = $this->pick($this->transitions[$key]); + return $pick[0]; } private function pick($options) { @@ -142,84 +128,62 @@ class ChatLib { return $options[$min_index]; } - private function addTransition($state, $next) { - $ctx = $this->generalize($state); - $cmb = $this->generalize([$next]); - if (!isset($this->transitions[$ctx])) { - $this->transitions[$ctx] = []; + private function addTransition($tokens, $next) { + $key = $this->makeKey($tokens); + if (!isset($this->transitions[$key])) { + $this->transitions[$key] = []; } - if (!isset($this->transitions[$ctx][$cmb])) { - $this->transitions[$ctx][$cmb] = 1; + if (!isset($this->transitions[$key][$next])) { + $this->transitions[$key][$next] = 1; } else { - ++$this->transitions[$ctx][$cmb]; + ++$this->transitions[$key][$next]; } } - private function addExample($context, $token) { - $cmb = $this->generalize([$token]); - if (!isset($this->examples[$cmb])) { - $this->examples[$cmb] = []; - } - if (!isset($this->examples[$cmb][$token])) { - $this->examples[$cmb][$token] = 1; - } else { - ++$this->examples[$cmb][$token]; + private function splitText($text) { + if (trim($text) === '') return []; + return preg_split('/\s+/u', $text); + } + + private function makeKey($tokens) { + $key = $this->joinText(array_slice($tokens, $this->size * -1)); + $key = mb_strtolower($key); + $key = str_replace(['.', ',', ':', ';', '!', '?', '^', '+', '-', '"', "'", '(', ')', '[', ']'], '', $key); + $key = preg_replace('/\d+/u', '0', $key); + foreach ($this->categories as $category => $patterns) { + $key = preg_replace($patterns, $category, $key); } + return $key; } - private function tokenize($str) { - return array_values(array_filter(preg_split('/\b/u', $str), function($token) { - if ($token === '') return false; - if (preg_match('/cheer\d+/u', strtolower($token))) return false; - return true; - })); + private function joinText($tokens) { + return implode(' ', $tokens); } - private function generalize($tokens) { - $str = ''; - foreach ($tokens as $token) { - $replaced = preg_replace('/\d+/u', '0', $token); - $replaced = preg_replace('/\s+/u', ' ', $replaced); - $replaced = preg_replace('/(.)\1{2,}/u', '$1$1', $replaced); - $replaced = strtolower($replaced); - foreach ($this->aliases as $canonical => $variants) { - if (in_array($replaced, $variants)) { - $replaced = $canonical; - break; - } - if ($replaced === $canonical) { - break; - } - } - $str .= $replaced; - } - foreach ($this->categories as $category => $patterns) { - $str = preg_replace($patterns, $category, $str); - } - return $str; + private function untokenize($tokens) { + return $this->joinText(array_slice($tokens, $this->size)); } - private function exampleOf($pick, $context) { - if (!isset($this->examples[$pick[0]])) { - return $pick[0]; + private function tokenize($text) { + $tokens = $this->splitText($text); + $combined = array_merge(array_fill(0, $this->size, ' '), $tokens); + if (!empty($tokens)) { + $combined[] = ' '; } - if (isset($this->examples[$pick[0]])) { - $example = $this->pick($this->examples[$pick[0]]); - return $example[0]; + return $combined; + } + + private function tokenizeWithContext($text, $context) { + $combined = $this->tokenize($text); + $context_tokens = array_slice($this->splitText($context), $this->size * -1 + 1); + for ($i = 0; $i < count($context_tokens); ++$i) { + $combined[$this->size - $i - 2] = $context_tokens[count($context_tokens) - $i - 1]; } - return $pick[0]; + return $combined; } private $size; private $transitions = []; - private $examples = []; - - private $aliases = [ - 'chest' => ['kiste'], - 'einen' => ['n', 'nen'], - 'musik' => ['mukke'], - 'schade' => ['schad', 'schaade'], - ]; private $categories = [ 'fail' => [ diff --git a/app/Models/ChatLog.php b/app/Models/ChatLog.php index 2d43b2f..cad1bfe 100644 --- a/app/Models/ChatLog.php +++ b/app/Models/ChatLog.php @@ -25,8 +25,24 @@ class ChatLog extends Model { return TokenizedMessage::fromLog($this); } + public function isReply() { + return !empty($this->tags['reply-parent-msg-body']); + } + + public function getReplyParent() { + return str_replace('\\s', ' ', $this->tags['reply-parent-msg-body']); + } + + public function getReplyParentUser() { + return $this->tags['reply-parent-display-name']; + } + + public function getText() { + return $this->params[1]; + } + public function getTextWithoutEmotes() { - $text = $this->text_content; + $text = $this->params[1]; if (isset($this->tags['emotes']) && !empty($this->tags['emotes'])) { $emotes = explode('/', $this->tags['emotes']); foreach ($emotes as $emote) { @@ -41,6 +57,13 @@ class ChatLog extends Model { return trim(preg_replace('/\s+/', ' ', $text)); } + public function getTextWithoutReply() { + if ($this->isReply()) { + return mb_substr($this->params[1], mb_strlen($this->getReplyParentUser()) + 2); + } + return $this->params[1]; + } + public function evaluate() { $this->evaluateUser(); $this->evaluateChannel(); @@ -62,7 +85,7 @@ class ChatLog extends Model { } else { $this->type = 'dm'; } - $this->text_content = $this->params[1]; + $this->text_content = $this->getTextWithoutReply(); $this->detectLanguage(); $tokenized = $this->tokenize(); if ($tokenized->isSpammy()) { diff --git a/app/TwitchBot/TokenizedMessage.php b/app/TwitchBot/TokenizedMessage.php index b460020..72817c1 100644 --- a/app/TwitchBot/TokenizedMessage.php +++ b/app/TwitchBot/TokenizedMessage.php @@ -11,10 +11,13 @@ class TokenizedMessage { public function __construct($text, $tags = []) { $this->text = trim($text); $this->tags = $tags; + if (isset($tags['reply-parent-display-name'])) { + $this->text = mb_substr($text, mb_strlen($tags['reply-parent-display-name']) + 2); + } $this->raw = strtolower(preg_replace('/[^\w]/u', '', $this->text)); $this->tokens = array_values(array_map('trim', array_filter(preg_split('/\b/u', strtolower($this->text))))); - $this->emoteless = $this->text; + $this->emoteless = $text; if (isset($this->tags['emotes']) && !empty($this->tags['emotes'])) { $emotes = explode('/', $this->tags['emotes']); foreach ($emotes as $emote) { @@ -253,7 +256,7 @@ class TokenizedMessage { if ($this->contains(['€', '$', '@', '://'])) { return true; } - if ($this->containsRaw(['followers', 'promotion', 'viewers'])) { + if ($this->containsRaw(['follow', 'promotion', 'viewer'])) { return true; } if ($this->containsRaw('horsti')) { -- 2.39.2 From c8a2f8ca5a615f5e07e35fb3926ef9d08ddb9cad Mon Sep 17 00:00:00 2001 From: Daniel Karbach Date: Fri, 28 Jun 2024 16:49:20 +0200 Subject: [PATCH 12/14] add context to adlibs --- app/TwitchBot/TokenizedMessage.php | 11 ++++++++--- app/TwitchBot/TwitchChatBot.php | 20 +++++++++++++------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/app/TwitchBot/TokenizedMessage.php b/app/TwitchBot/TokenizedMessage.php index 72817c1..f218e7a 100644 --- a/app/TwitchBot/TokenizedMessage.php +++ b/app/TwitchBot/TokenizedMessage.php @@ -50,6 +50,11 @@ class TokenizedMessage { } + public function getText() { + return $this->text; + } + + public function contains($text) { return Str::contains($this->text, $text); } @@ -286,7 +291,7 @@ class TokenizedMessage { $this->classification = 'cmd'; } else if ($this->isShort() && ($this->hasTokenThatStartsOrEndsWith(['gg']) || $this->hasEmoteThatEndsWith(['gg']))) { $this->classification = 'gg'; - } else if ($this->isShort() && $this->containsRaw(['glgl', 'glhf', 'goodluck', 'hfgl', 'vielglück'])) { + } else if ($this->isShort() && ($this->containsRaw(['glgl', 'glhf', 'goodluck', 'hfgl', 'vielglück']) || $this->hasToken('gl'))) { $this->classification = 'gl'; } else if ($this->hasToken(['danke', 'thanks', 'thx', 'ty']) && !$this->hasToken(['nah', 'nee', 'nein', 'no'])) { $this->classification = 'thx'; @@ -348,9 +353,9 @@ class TokenizedMessage { $this->hasConsecutiveTokens(['how', 'much']) || $this->hasConsecutiveTokens(['wie', 'viele']) ) { - return ['yes', 'no', 'kappa', 'lol', 'wtf', 'number']; + return ['yes', 'no', 'kappa', 'wtf', 'number']; } - return ['yes', 'no', 'kappa', 'lol', 'wtf']; + return ['yes', 'no', 'kappa', 'wtf']; case 'rage': return ['kappa', 'lol', 'rage']; case 'wtf': diff --git a/app/TwitchBot/TwitchChatBot.php b/app/TwitchBot/TwitchChatBot.php index 6ec517e..700de25 100644 --- a/app/TwitchBot/TwitchChatBot.php +++ b/app/TwitchBot/TwitchChatBot.php @@ -80,7 +80,7 @@ class TwitchChatBot extends TwitchBot { return; } $text = $this->contextualMsg($channel); - if ($this->shouldAdlib($channel)) { + if (!$text && $this->shouldAdlib($channel)) { $this->performAdlib($channel); return; } @@ -258,7 +258,8 @@ class TwitchChatBot extends TwitchBot { private function performAdlib(Channel $channel) { $db = $this->getChatlibDatabase($channel); - $text = $db->generate(); + $latest_msg = $this->getLatestMessage($channel); + $text = $db->generate($latest_msg->getText()); $this->tagChannelWrite($channel); $this->sendIRCMessage(IRCMessage::privmsg($channel->twitch_chat, $text)); $log = new ChatBotLog(); @@ -306,12 +307,10 @@ class TwitchChatBot extends TwitchBot { $tokenized = $msg->tokenize(); if (!ChatLog::isKnownBot($msg->nick) && !$tokenized->isSpammy()) { - $this->notes[$channel->id]['latest_msgs'][] = $tokenized; - if (count($this->notes[$channel->id]['latest_msgs']) > 10) { - array_shift($this->notes[$channel->id]['latest_msgs']); - } + $this->noteChannelMessage($channel, $tokenized); } if ($this->isDirectedAtMe($msg->getText()) && $this->shouldRespond($channel)) { + $this->noteChannelMessage($channel, $tokenized); $this->notes[$channel->id]['wait_msgs'] = 0; $this->notes[$channel->id]['wait_time'] = 0; $response = $tokenized->getResponseCategory(); @@ -321,6 +320,13 @@ class TwitchChatBot extends TwitchBot { } } + private function noteChannelMessage(Channel $channel, TokenizedMessage $tokenized) { + $this->notes[$channel->id]['latest_msgs'][] = $tokenized; + if (count($this->notes[$channel->id]['latest_msgs']) > 10) { + array_shift($this->notes[$channel->id]['latest_msgs']); + } + } + private function tagChannelWrite(Channel $channel) { $this->getNotes($channel); $this->notes[$channel->id]['last_write'] = time(); @@ -335,7 +341,7 @@ class TwitchChatBot extends TwitchBot { } private function getLatestMessage(Channel $channel) { - $this->getNotes($channel); + $notes = $this->getNotes($channel); if (!empty($notes['latest_msgs'])) { return $notes['latest_msgs'][count($notes['latest_msgs']) - 1]; } -- 2.39.2 From af4f2e9450392e41e4a0bfb3247fe3b4435f03c7 Mon Sep 17 00:00:00 2001 From: Daniel Karbach Date: Fri, 28 Jun 2024 17:27:24 +0200 Subject: [PATCH 13/14] whispers --- app/Console/Commands/TwitchAuth.php | 2 +- app/Models/ChatLog.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/Console/Commands/TwitchAuth.php b/app/Console/Commands/TwitchAuth.php index e46c118..4b185c6 100644 --- a/app/Console/Commands/TwitchAuth.php +++ b/app/Console/Commands/TwitchAuth.php @@ -33,7 +33,7 @@ class TwitchAuth extends Command { if (!$token) { $token = new TwitchToken(); $token->nick = $this->argument('nick'); - $token->scope = ['chat:read', 'chat:edit']; + $token->scope = ['chat:read', 'chat:edit', 'whispers:read', 'whispers:edit']; } $url = $token->getAuthUrl(); $this->line('Please visit '.$url); diff --git a/app/Models/ChatLog.php b/app/Models/ChatLog.php index cad1bfe..49a57c2 100644 --- a/app/Models/ChatLog.php +++ b/app/Models/ChatLog.php @@ -77,7 +77,7 @@ class ChatLog extends Model { return; } - if ($this->command == 'PRIVMSG') { + if ($this->command == 'PRIVMSG' || $this->command == 'WHISPER') { if (static::isKnownBot($this->nick)) { $this->type = 'bot'; } else if (substr($this->params[0], 0, 1) == '#') { -- 2.39.2 From b51ffd97a5ec084ad2e4e7804cb047b464a2b58b Mon Sep 17 00:00:00 2001 From: Daniel Karbach Date: Fri, 28 Jun 2024 18:40:28 +0200 Subject: [PATCH 14/14] respond to whispers --- app/Console/Commands/TwitchAuth.php | 2 +- app/TwitchBot/IRCMessage.php | 11 ++++++++++ app/TwitchBot/TwitchBot.php | 33 +++++++++++++++++++++++++++++ app/TwitchBot/TwitchChatBot.php | 5 +++++ 4 files changed, 50 insertions(+), 1 deletion(-) diff --git a/app/Console/Commands/TwitchAuth.php b/app/Console/Commands/TwitchAuth.php index 4b185c6..94d8ada 100644 --- a/app/Console/Commands/TwitchAuth.php +++ b/app/Console/Commands/TwitchAuth.php @@ -33,7 +33,7 @@ class TwitchAuth extends Command { if (!$token) { $token = new TwitchToken(); $token->nick = $this->argument('nick'); - $token->scope = ['chat:read', 'chat:edit', 'whispers:read', 'whispers:edit']; + $token->scope = ['chat:read', 'chat:edit', 'whispers:read', 'user:manage:whispers']; } $url = $token->getAuthUrl(); $this->line('Please visit '.$url); diff --git a/app/TwitchBot/IRCMessage.php b/app/TwitchBot/IRCMessage.php index 0f73c06..b986847 100644 --- a/app/TwitchBot/IRCMessage.php +++ b/app/TwitchBot/IRCMessage.php @@ -140,6 +140,13 @@ class IRCMessage { return TokenizedMessage::fromIRC($this); } + public static function capReq($cap) { + $msg = new IRCMessage(); + $msg->command = 'CAP REQ'; + $msg->params[] = $cap; + return $msg; + } + public static function join($channels) { $msg = new IRCMessage(); $msg->command = 'JOIN'; @@ -199,6 +206,10 @@ class IRCMessage { return $this->command == 'PRIVMSG'; } + public function isWhisper() { + return $this->command == 'WHISPER'; + } + public function isOwner() { return substr($this->getPrivMsgTarget(), 1) == $this->nick; } diff --git a/app/TwitchBot/TwitchBot.php b/app/TwitchBot/TwitchBot.php index 63f4236..242ac92 100644 --- a/app/TwitchBot/TwitchBot.php +++ b/app/TwitchBot/TwitchBot.php @@ -118,6 +118,10 @@ class TwitchBot { $this->handlePrivMsg($msg); return; } + if ($msg->isWhisper()) { + $this->handleWhisper($msg); + return; + } if ($msg->isNotice() && $msg->getText() == 'Login authentication failed') { $this->logger->notice('login failed, refreshing access token'); $this->token->refresh(); @@ -130,6 +134,11 @@ class TwitchBot { $this->ready = true; return; } + if ($msg->command == 'GLOBALUSERSTATE') { + // receive own user metadata + $this->handleUserState($msg); + return; + } } public function getMessageChannel(IRCMessage $msg) { @@ -146,6 +155,15 @@ class TwitchBot { public function handlePrivMsg(IRCMessage $msg) { } + public function handleUserState(IRCMessage $msg) { + if (isset($msg->tags['user-id'])) { + $this->user_id = $msg->tags['user-id']; + } + } + + public function handleWhisper(IRCMessage $msg) { + } + public function login() { $this->ws->send('PASS oauth:'.$this->token->access); $this->ws->send('NICK '.$this->nick); @@ -172,6 +190,20 @@ class TwitchBot { $this->last_contact = time(); } + public function sendWhisper($to, $msg) { + $this->logger->info('sending whisper to '.$to.': '.$msg); + try { + $response = $this->token->request()->post('/whispers?from_user_id='.$this->user_id.'&to_user_id='.$to, [ + 'message' => $msg, + ]); + if (!$response->successful()) { + $this->logger->error('sending whisper to '.$to.': '.$response->status()); + } + } catch (\Exception $e) { + $this->logger->error('sending whisper to '.$to.': '.$e->getMessage()); + } + } + protected function listenCommands() { $this->getLoop()->addPeriodicTimer(1, function () { @@ -191,6 +223,7 @@ class TwitchBot { private $nick; private $token; + private $user_id = ''; private $connector; private $ws; diff --git a/app/TwitchBot/TwitchChatBot.php b/app/TwitchBot/TwitchChatBot.php index 700de25..7151b2b 100644 --- a/app/TwitchBot/TwitchChatBot.php +++ b/app/TwitchBot/TwitchChatBot.php @@ -46,6 +46,11 @@ class TwitchChatBot extends TwitchBot { $this->tagChannelRead($channel, $msg); } + public function handleWhisper(IRCMessage $msg) { + $text = $this->chatlib->generate($msg->getText()); + $this->sendWhisper($msg->tags['user-id'], $text); + } + public function getChatlibDatabase(Channel $channel) { return $this->chatlib; } -- 2.39.2