- private function addExample($context, $token) {
- $cmb = $this->generalize([$token]);
- if (!isset($this->examples[$cmb])) {
- $this->examples[$cmb] = [];
- }
- if (!isset($this->examples[$cmb][$token])) {
- $this->examples[$cmb][$token] = 1;
- } else {
- ++$this->examples[$cmb][$token];
+ private function splitText($text) {
+ if (trim($text) === '') return [];
+ return preg_split('/\s+/u', $text);
+ }
+
+ private function makeKey($tokens) {
+ $key = $this->joinText(array_slice($tokens, $this->size * -1));
+ $key = mb_strtolower($key);
+ $key = str_replace(['.', ',', ':', ';', '!', '?', '^', '+', '-', '"', "'", '(', ')', '[', ']'], '', $key);
+ $key = preg_replace('/\d+/u', '0', $key);
+ foreach ($this->categories as $category => $patterns) {
+ $key = preg_replace($patterns, $category, $key);