setup basic FR preprocessing

.gitignore

@@ -47,3 +47,4 @@ MANIFEST
 
 # Per-project virtualenvs
 .virtualenv/
+.python-version

setup.cfg

@@ -29,6 +29,7 @@ setup_requires = pyscaffold>=3.1a0,<3.2a0
 # Add here dependencies of your project (semicolon/line-separated), e.g.
 install_requires = pandas
                    swifter
+                   num2words==0.5.9
 # The usage of test_requires is discouraged, see `Dependency Management` docs
 # tests_require = pytest; pytest-cov
 # Require a specific Python version, e.g. Python 2.7 or >= 3.4

src/corporacreator/preprocessors/fr.py

@@ -1,3 +1,27 @@
+import re
+
+from corporacreator.utils import maybe_normalize, replace_numbers, FIND_PUNCTUATIONS_REG, FIND_MULTIPLE_SPACES_REG
+
+FIND_ORDINAL_REG = re.compile(r"(\d+)([ème|éme|ieme|ier|iere]+)")
+
+
+FR_NORMALIZATIONS = [
+    [re.compile(r'(^|\s)(\d+)\s(0{3})(\s|\.|,|\?|!|$)'), r'\1\2\3\4'],  # "123 000 …" => "123000 …"
+    [re.compile(r'(^|\s)km(\s|\.|,|\?|!|$)'), r'\1 kilomètres \2'],
+    [re.compile(r'(^|\s)0(\d)(\s|\.|,|\?|!|$)'), r'\1zéro \2 \3'],
+    ['%', ' pourcent'],
+    [re.compile(r'(^|\s)\+(\s|\.|,|\?|!|$)'), r'\1 plus \2'],
+    [re.compile(r'(\d+)\s?m(?:2|²)(\s|\.|,|\?|!|$)'), r'\1 mètre carré\2'],
+    [re.compile(r'(^|\s|/)m(?:2|²)(\s|\.|,|\?|!|$)'), r' mètre carré\2'],
+    [re.compile(r'(^|\s)(\d+),(\d{2})\s?€(\s|\.|,|\?|!|$)'), r'\1\2 euros \3 \4'],
+    [re.compile(r'\s?€(.+)'), r' euros\1'],
+    [re.compile(r'\s?€$'), r' euros'],
+    [re.compile(r'(^| )(n)(?:°|º|°)(\s)?', flags=re.IGNORECASE), r'\1\2uméro '],
+    [re.compile(r'(^|\s)(\d+)h(\d*)(\s|\.|,|$)'), r'\1\2 heure \3\4'],
+    [re.compile(r'(^|\s)(\d+)h(\s|\.|,|$)'), r'\1\2 heure \3'],
+]
+
+
 def fr(client_id, sentence):
     """Cleans up the passed sentence, removing or reformatting invalid data.
 
@@ -8,5 +32,9 @@ def fr(client_id, sentence):
     Returns:
       (str): Cleaned up sentence. Returning None or a `str` of whitespace flags the sentence as invalid.
     """
-    # TODO: Clean up fr data
-    return sentence
+    text = maybe_normalize(sentence, mapping=FR_NORMALIZATIONS)
+    text = replace_numbers(text, locale='fr', ordinal_regex=FIND_ORDINAL_REG)
+    text = text.replace('’', "'").replace('\u00A0', ' ')
+    text = FIND_PUNCTUATIONS_REG.sub(' ', text)
+    text = FIND_MULTIPLE_SPACES_REG.sub(' ', text)
+    return text.strip().lower()

src/corporacreator/utils.py

@@ -0,0 +1,51 @@
+import re
+from typing import Pattern
+
+from num2words import num2words
+
+
+NUMS_REGEX = re.compile(r"(\d+,?\u00A0?\d+)|(\d+\w+)|(\d)+")
+FIND_MULTIPLE_SPACES_REG = re.compile(r'\s{2,}')
+FIND_PUNCTUATIONS_REG = re.compile(r"[/°\-,;!?.()\[\]*…—]")
+
+
+def get_numbers(text):
+    return NUMS_REGEX.split(text)
+
+
+def replace_numbers(inp: str, locale: str, ordinal_regex: Pattern = None):
+    finalinp = ''
+    for e in get_numbers(inp):
+        if not e:
+            continue
+        newinp = e
+        try:
+            ee = ''.join(e.split())
+            if int(e) >= 0:
+                newinp = num2words(int(ee), lang=locale)
+        except ValueError:
+            try:
+                ee = ''.join(e.replace(',', '.').split())
+                if float(ee):
+                    newinp = num2words(float(ee), lang=locale)
+            except ValueError:
+                if ordinal_regex:
+                    matches = ordinal_regex.match(e)
+                    if matches:
+                        newinp = num2words(int(matches.group(1)), ordinal=True, lang=locale)
+
+        finalinp += newinp
+
+    return finalinp
+
+
+def maybe_normalize(value: str, mapping):
+    for norm in mapping:
+        if type(norm[0]) == str:
+            value = value.replace(norm[0], norm[1])
+        elif isinstance(norm[0], Pattern):
+            value = norm[0].sub(norm[1], value)
+        else:
+            print('UNEXPECTED', type(norm[0]), norm[0])
+
+    return value

tests/test_preprocessors.py

@@ -0,0 +1,21 @@
+import pytest
+
+from corporacreator import preprocessors
+
+
+@pytest.mark.parametrize('locale, client_id, sentence, expected', [
+    ('fr', '*', 'Faisons donc attention à utiliser les bons mots.', 'faisons donc attention à utiliser les bons mots'),
+    ('fr', '*', "bah 98%", "bah quatre vingt dix huit pourcent"),
+    ('fr', '*', "prix au m2", "prix au mètre carré"),
+    ('fr', '*', "prix au m²", "prix au mètre carré"),
+    ('fr', '*', "10 m²", "dix mètre carré"),
+    ('fr', '*', "2éme page", "deuxième page"),
+    ('fr', '*', "donc, ce sera 299 € + 99 €", "donc ce sera deux cent quatre vingt dix neuf euros plus quatre vingt dix neuf euros"),
+    ('fr', '*', "ok pour 18h", "ok pour dix huit heure"),
+    ('fr', '*', '2 0 200', "deux zéro deux cents"),
+    ('fr', '*', 'rue Coq-Héron au nº13', "rue coq héron au numéro treize"),
+    ('fr', '*', "En comparaison, la Lune orbite en moyenne à 390 000 km de la Terre", "en comparaison la lune orbite en moyenne à trois cent quatre vingt dix mille kilomètres de la terre"),
+])
+def test_preprocessor(locale, client_id, sentence, expected):
+    preprocessor = getattr(preprocessors, locale.replace('-', ''))
+    assert expected == preprocessor(client_id, preprocessors.common(sentence))