- private function generalize($tokens) {
- $str = '';
- foreach ($tokens as $token) {
- $replaced = preg_replace('/\d+/u', '0', $token);
- $replaced = preg_replace('/\s+/u', ' ', $replaced);
- $replaced = preg_replace('/(.)\1{2,}/u', '$1$1', $replaced);
- $replaced = strtolower($replaced);
- foreach ($this->aliases as $canonical => $variants) {
- if (in_array($replaced, $variants)) {
- $replaced = $canonical;
- break;
- }
- if ($replaced === $canonical) {
- break;
- }
- }
- $str .= $replaced;
- }
+ private function makeKey($tokens) {
+ $key = $this->joinText(array_slice($tokens, $this->size * -1));
+ $key = mb_strtolower($key);
+ $key = str_replace(['.', ',', ':', ';', '!', '?', '^', '+', '-', '"', "'", '(', ')', '[', ']'], '', $key);
+ $key = preg_replace('/\d+/u', '0', $key);