diff --git a/bin/demeuk.py b/bin/demeuk.py
index 117ba1c..8c38ffe 100755
--- a/bin/demeuk.py
+++ b/bin/demeuk.py
@@ -111,6 +111,8 @@
becomes u, ç becomes c.
--trim Enables removing newlines representations from end and beginning. Newline
representations detected are '\\n', '\\r', '\n', '\r', '
', and '
'.
+ --transliterate Transliterate a strings, for example "ipsum" becomes "իպսում". Language is iso
+ 2 letter code. Examples: ru, sr, ua
Add modules (Modify a line, but keep the original as well):
--add-lower If a line contains a capital letter this will add the lower case variant
@@ -169,10 +171,11 @@
from nltk import str2tuple
from nltk.tokenize import WhitespaceTokenizer
from tqdm import tqdm
+from transliterate import translit
from unidecode import unidecode
-version = '4.5.1'
+version = '4.6.0'
# Search from start to finish for the string $HEX[], with block of a-f0-9 with even number
# of hex chars. The first match group is repeated.
@@ -680,6 +683,23 @@ def clean_cut(line, delimiters, fields):
return False, line
+def clean_transliterate(line, language):
+ """Transliterate a string
+
+ Params:
+ line (Unicode)
+ language (str)
+
+ Returns:
+ line (Unicode)
+ """
+ cleaned_line = translit(line, language, reversed=True)
+ if line != cleaned_line:
+ return True, cleaned_line
+ else:
+ return False, line
+
+
def clean_non_ascii(line):
"""Replace non ascii chars with there ascii representation.
@@ -1129,6 +1149,12 @@ def clean_up(lines):
if status and config['debug']:
log.append(f'Clean_umlaut; umlaut replaced; {line_decoded}{linesep}')
+ # Transliterate
+ if config.get('transliterate') and not stop:
+ status, line_decoded = clean_transliterate(line_decoded, config.get('transliterate'))
+ if status and config['debug']:
+ log.append(f'Clean_transliterate; translitatered; {line_decoded}{linesep}')
+
# Replace non-ascii
if config.get('non-ascii') and not stop:
status, line_decoded = clean_non_ascii(line_decoded)
@@ -1409,6 +1435,7 @@ def main():
'umlaut': False,
'non-ascii': False,
'title_case': False,
+ 'transliterate': False,
# Check
'length': False,
@@ -1543,6 +1570,9 @@ def main():
if arguments.get('--trim'):
config['trim'] = True
+ if arguments.get('--transliterate'):
+ config['transliterate'] = arguments.get('--transliterate')
+
# Check modules
if arguments.get('--check-min-length'):
config['check-length'] = True
diff --git a/docs/usage.rst b/docs/usage.rst
index 906552b..ac6eda6 100644
--- a/docs/usage.rst
+++ b/docs/usage.rst
@@ -418,6 +418,17 @@ https://pypi.org/project/Unidecode/
For example a line like 'kožušček' is replaced to kozuscek.
+transliterate
+~~~~~~~~~~~~~
+Replaces Cyrillic characters with their Latin equivalents. For example, жута becomes Žuta. To take this even further,
+combine it with --non-ascii to convert this to zuta.
+
+The follow languages are supported: ka, sr, l1, ru, mn, uk, mk, el, hy and bg
+
+--transliterate ru
+
+Check https://pypi.org/project/transliterate/ for more details.
+
lowercase
~~~~~~~~~~
Replace lines like 'Test Test Test' to 'test test test'. Basically lowercasing all
diff --git a/requirements.txt b/requirements.txt
index 85532a5..7ac56f4 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,3 +4,4 @@ nltk
ftfy
unidecode
tqdm
+transliterate
diff --git a/tests/conftest.py b/tests/conftest.py
index 0a2d6a6..1d46472 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -397,3 +397,7 @@
with open('testdata/input54', 'w') as file:
file.write(f'Golf Trip{linesep}')
file.write(f'Sequences{linesep}')
+
+with open('testdata/input55', 'w') as file:
+ file.write(f'здраво пријатељу{linesep}')
+ file.write(f'жута банана{linesep}')
diff --git a/tests/test_app.py b/tests/test_app.py
index f603ca8..4302f26 100644
--- a/tests/test_app.py
+++ b/tests/test_app.py
@@ -1003,3 +1003,19 @@ def test_infinite_loop():
assert 'Sequences' in filecontent
assert 'golf trip' in filecontent
assert 'sequences' in filecontent
+
+
+def test_transliterate():
+ testargs = [
+ 'demeuk', '-i', 'testdata/input55', '-o', 'testdata/output55', '-l', 'testdata/log55',
+ '--transliterate', 'sr', '--non-ascii'
+ ]
+
+ with patch.object(sys, 'argv', testargs):
+ main()
+
+ with open('testdata/output55') as f:
+ filecontent = f.read()
+
+ assert 'zdravo prijatelju' in filecontent
+ assert 'zuta banana' in filecontent