AlderconeStudio/carkov
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Bump version to 0.4; fix some changes to analyze/english

Browse Source
This commit is contained in:
Cassowary
2026-03-24 19:33:30 -07:00
parent d6e329ff39
commit 3fae360c2e
3 changed files with 5 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
carkov/__init__.py
View File

@ -4,4 +4,4 @@
# This is free software, see the included LICENSE for terms and conditions. # This is free software, see the included LICENSE for terms and conditions.
# #
version = '0.2.0' version = '0.4.0'

14
carkov/analyze/english.py
View File

@ -4,16 +4,8 @@
# This is free software, see the included LICENSE for terms and conditions. # This is free software, see the included LICENSE for terms and conditions.
# #
import nltk
from .abstract import AbstractAnalyzer from .abstract import AbstractAnalyzer
from .utils import english_sentence_split, english_sentence_tokenize
try:
nltk.sent_tokenize("foo bar")
except LookupError:
nltk.download('punkt')
class English(AbstractAnalyzer): class English(AbstractAnalyzer):
def __init__(self, order, filters=None): def __init__(self, order, filters=None):
@ -25,8 +17,8 @@ class English(AbstractAnalyzer):
chunks = corpus.split('\n\n') chunks = corpus.split('\n\n')
ret = [] ret = []
for chunk in chunks: for chunk in chunks:
ret = ret + nltk.sent_tokenize(chunk) ret = ret + english_sentence_split(chunk)
return ret return ret
def tokenize_segment(self, segment): def tokenize_segment(self, segment):
return list(nltk.word_tokenize(segment)) return list(english_sentence_tokenize(segment))

2
setup.cfg
View File

@ -1,6 +1,6 @@
[metadata] [metadata]
name = carkov name = carkov
version = 0.2.0 version = 0.4.0
description = A markov chainer library description = A markov chainer library
author = Aldercone Studio author = Aldercone Studio
author_email = alderconestudio@gmail.com author_email = alderconestudio@gmail.com