ChatLog::where('type', '=', 'chat')
->where('banned', '=', false)
->whereNotNull('evaluated_at')
+ ->where('created_at', '<', now()->sub(7, 'day'))
->chunk(5000, function ($msgs) use ($de, $en) {
foreach ($msgs as $msg) {
if ($msg->detected_language === 'de') {
});
$de->compile();
- $en->compile();
+ $de->saveAs('de');
- $this->line('');
- for ($i = 0; $i < 50; ++$i) {
- $this->line($de->generate());
- }
- $this->line('');
- for ($i = 0; $i < 50; ++$i) {
- $this->line($en->generate());
- }
+ $en->compile();
+ $en->saveAs('en');
return 0;
}
--- /dev/null
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\ChatLib;
+use Illuminate\Console\Command;
+
+class ChatlibGenerate extends Command {
+
+ /**
+ * The name and signature of the console command.
+ *
+ * @var string
+ */
+ protected $signature = 'chatlib:generate {which=de} {amount=50}';
+
+ /**
+ * The console command description.
+ *
+ * @var string
+ */
+ protected $description = 'Generates samples from a ChatLib database';
+
+ /**
+ * Execute the console command.
+ *
+ * @return int
+ */
+ public function handle() {
+ $db = new ChatLib();
+ $db->loadFrom($this->argument('which'));
+
+ $amount = intval($this->argument('amount'));
+ for ($i = 0; $i < $amount; ++$i) {
+ $this->line($db->generate());
+ }
+
+ return 0;
+ }
+
+}
+
+?>
namespace App\Models;
+use Illuminate\Support\Facades\Storage;
+
class ChatLib {
public function addMessage($msg) {
unset($this->transitions[$key]);
}
}
- echo 'size: ', number_format(strlen(json_encode($this->transitions)), 0), PHP_EOL;
}
public function generate($limit = 100) {
return $generated;
}
+ public function saveAs($name) {
+ $data = [
+ 'size' => $this->size,
+ 'transitions' => $this->transitions,
+ ];
+ Storage::disk('chatlib')->put($name.'.json', json_encode($data));
+ }
+
+ public function loadFrom($name) {
+ $data = json_decode(Storage::disk('chatlib')->get($name.'.json'), true);
+ $this->size = $data['size'];
+ $this->transitions = $data['transitions'];
+ }
+
private function index($arr) {
$result = [];
$sum = 0;
}
private function tokenize($str) {
- return array_values(array_filter(preg_split('/\b/u', $str)));
+ return array_values(array_filter(preg_split('/\b/u', $str), function($token) {
+ if (empty($token)) return false;
+ if (preg_match('/cheer\d+/u', strtolower($token))) return false;
+ return true;
+ }));
}
private function generalize($tokens) {
'root' => storage_path('app/alttp-spoilers'),
],
+ 'chatlib' => [
+ 'driver' => 'local',
+ 'root' => storage_path('app/chatlib'),
+ ],
+
'media' => [
'driver' => 'local',
'root' => storage_path('app/media'),