mirror of
https://github.com/Findus23/se-simulator.git
synced 2024-09-19 15:53:45 +02:00
13 lines
358 B
Python
13 lines
358 B
Python
import markovify
|
|
from nltk.tokenize.moses import MosesDetokenizer, MosesTokenizer
|
|
|
|
tokenizer = MosesTokenizer()
|
|
detokenizer = MosesDetokenizer()
|
|
|
|
|
|
class POSifiedText(markovify.Text):
|
|
def word_split(self, sentence):
|
|
return tokenizer.tokenize(sentence)
|
|
|
|
def word_join(self, words):
|
|
return detokenizer.detokenize(words, return_str=True)
|