evaluate_stems.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read vocabs from solr "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "417673\n"
     ]
    }
   ],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "with open('lemma/wordlemma-stat-fr.csv', 'r', encoding=\"utf8\") as file:\n",
    "    d = {line.rstrip('\\n').split(',')[0]:int(line.rstrip('\\n').split(',')[1]) for line in file.readlines()}\n",
    "    fr_vocab_lemmas = defaultdict(lambda:0, d)\n",
    "    print(fr_vocab_lemmas['a'])\n",
    "    print(fr_vocab_lemmas['ferguson'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'numFound': 1, 'ids': ['50554053']}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from solr_client import SolrClient\n",
    "solr1 = SolrClient('localhost',8986)\n",
    "solr2 = SolrClient('be-plw-tst-0050',8983)\n",
    "solr2.auth(\"solr\",\"SolrRocks\")\n",
    "solr1.auth(\"solr\",\"SolrRocks\")\n",
    "#solr2 = SolrClient('localhost',8981)\n",
    "solr2.search('textsearch', 'STARTVEILIGHEIDSSCHAKELAAR','nl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stemmers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting nl-core-news-sm==3.4.0\n",
      "  Downloading https://github.com/explosion/spacy-models/releases/download/nl_core_news_sm-3.4.0/nl_core_news_sm-3.4.0-py3-none-any.whl (12.8 MB)\n",
      "Requirement already satisfied: spacy<3.5.0,>=3.4.0 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from nl-core-news-sm==3.4.0) (3.4.0)\n",
      "Requirement already satisfied: numpy>=1.15.0 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (1.21.4)\n",
      "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (2.4.3)\n",
      "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.10.0,>=1.7.4 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (1.9.1)\n",
      "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (2.0.7)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (2.21.0)\n",
      "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (1.0.3)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (0.9.1)\n",
      "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (3.3.0)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (4.43.0)\n",
      "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.9 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (3.0.9)\n",
      "Requirement already satisfied: pathy>=0.3.5 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (0.6.2)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (1.0.7)\n",
      "Requirement already satisfied: thinc<8.2.0,>=8.1.0 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (8.1.0)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (21.3)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: You are using pip version 21.3.1; however, version 22.2 is available.\n",
      "You should consider upgrading via the 'c:\\Projects\\tvh-search-labeling\\venv\\Scripts\\python.exe -m pip install --upgrade pip' command.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: setuptools in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (59.1.0)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (2.0.6)\n",
      "Requirement already satisfied: jinja2 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (2.10)\n",
      "Requirement already satisfied: typer<0.5.0,>=0.3.0 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (0.4.2)\n",
      "Requirement already satisfied: typing-extensions<4.2.0,>=3.7.4 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (3.10.0.2)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (3.0.6)\n",
      "Requirement already satisfied: zipp>=0.5 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from catalogue<2.1.0,>=2.0.6->spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (3.8.1)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from packaging>=20.0->spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (2.4.6)\n",
      "Requirement already satisfied: smart-open<6.0.0,>=5.2.1 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from pathy>=0.3.5->spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (5.2.1)\n",
      "Requirement already satisfied: urllib3<1.25,>=1.21.1 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (1.24.1)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (3.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (2019.6.16)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (2.8)\n",
      "Requirement already satisfied: blis<0.8.0,>=0.7.8 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from thinc<8.2.0,>=8.1.0->spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (0.7.8)\n",
      "Requirement already satisfied: click<9.0.0,>=7.1.1 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from typer<0.5.0,>=0.3.0->spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (8.1.3)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from jinja2->spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (1.1.1)\n",
      "Requirement already satisfied: colorama in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from click<9.0.0,>=7.1.1->typer<0.5.0,>=0.3.0->spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (0.4.4)\n",
      "Requirement already satisfied: importlib-metadata in c:\\projects\\tvh-search-labeling\\venv\\lib\\site-packages (from click<9.0.0,>=7.1.1->typer<0.5.0,>=0.3.0->spacy<3.5.0,>=3.4.0->nl-core-news-sm==3.4.0) (4.12.0)\n",
      "Installing collected packages: nl-core-news-sm\n",
      "Successfully installed nl-core-news-sm-3.4.0\n",
      "[+] Download and installation successful\n",
      "You can now load the package via spacy.load('nl_core_news_sm')\n"
     ]
    }
   ],
   "source": [
    "!python -m spacy download nl_core_news_sm\n",
    "!python -m spacy download fr_core_news_md"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trou\n",
      "long\n",
      "trous\n",
      "long-metrag\n",
      "accessoires\n",
      "accessoire\n",
      "accessoire\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "from solr_client import SolrClient\n",
    "from stemmers import Stemmer\n",
    "from stemmers import SpacyStemmer\n",
    "from stemmers import TruncateStem\n",
    "\n",
    "\n",
    "    \n",
    "solr1 = SolrClient('localhost',8986)\n",
    "\n",
    "#French stemmers\n",
    "toy_stemmer = Stemmer(solr1, index = 'textsearch', fieldtype='query', field = 'text_nlp_fr_z', name = 'FR nlp')\n",
    "lightFr_stemmer = Stemmer(solr1, index = 'textsearch', fieldtype='query', field = 'text_test_fr_z', name = 'FR LightFr')\n",
    "porterFr_stemmer = Stemmer(solr1, index = 'textsearch', fieldtype='fieldvalue', field = 'text_test_fr_z', name = 'FR Porter')\n",
    "spacy_stemmer = SpacyStemmer('fr_core_news_md', name = 'FR spicy')\n",
    "\n",
    "#choose etalon stemmer for operations when building vocab\n",
    "etalon_stemmer = spacy_stemmer\n",
    "\n",
    "# Dutch stemmers\n",
    "porterNL_stemmer = Stemmer(solr1, index = 'textsearch', fieldtype='query', field = 'text_test_nl_z', name='NL Porter')\n",
    "spacyNL_stemmer = SpacyStemmer('nl_core_news_sm', name='NL spicy')\n",
    "nl_stemmer = Stemmer(solr1, index = 'textsearch', fieldtype='query', field = 'text_nlp_nl_z', name='NL nlp')\n",
    "\n",
    "# Truncate stemmers (remove last N characters)\n",
    "trunc_stemmers = [TruncateStem(i) for i in list(range(1, 7))]\n",
    "\n",
    "print(toy_stemmer.stem_word(\"trous\"))\n",
    "print(spacy_stemmer.stem_word(\"long-métrage\"))\n",
    "print(lightFr_stemmer.stem_word(\"trous\"))\n",
    "print(porterFr_stemmer.stem_word(\"long-métrage\"))\n",
    "\n",
    "print(porterNL_stemmer.stem_word(\"accessoires\"))\n",
    "print(spacyNL_stemmer.stem_word(\"accessoires\"))\n",
    "print(nl_stemmer.stem_word(\"accessoires\"))\n",
    "\n",
    "#Stemmers to test\n",
    "#stemmers=[porterNL_stemmer, spicyNL_stemmer, nl_stemmer]\n",
    "stemmers=[toy_stemmer, spacy_stemmer, lightFr_stemmer, porterFr_stemmer]\n",
    "#stemmers=[spacy_stemmer]\n",
    "#stemmers=[]\n",
    "\n",
    "stemmers+=trunc_stemmers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Weight function\n",
    "\n",
    "function returning weight by word frequency\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def get_word_weight(word):\n",
    "    if word in fr_vocab_lemmas.keys():\n",
    "        lemma = word\n",
    "    else:\n",
    "        lemma = etalon_stemmer.stem_word(word)\n",
    "        \n",
    "    if fr_vocab_lemmas[lemma] > 1000:\n",
    "        return 10\n",
    "    elif fr_vocab_lemmas[lemma] > 0:\n",
    "        return 1\n",
    "    return 0.1\n",
    "\n",
    "print(get_word_weight(\"barre\"))\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "### Over-Stemming vs Under-Stemming calculation\n",
    "\n",
    "Two very useful metrics to quantify a stemmer with are the over and under stemming errors. Under-stemming is when two related words do not reduce to the same stem. We saw that using the previous stemmer, the word *tasting* conflated to the stem *tast* whereas the word *taste* conflated to the stem *taste*. The two words are related inflections but the stemmer doesn't reduce the two to the same.\n",
    "\n",
    "Similarly, consider the words *red* and *ring*. These two are totally unrelated words. However, using our stemmer, these two are conflated to the same stem i.e. *r*. This is over-stemming i.e. reducing two unrelated words into a same stem.\n",
    "\n",
    "### Paice's Method\n",
    "\n",
    "Counting over-stemming and under-stemming errors seems to be a very useful evaluation scheme for stemmers. After all, this method doesn't need the concept of a correct stem and instead only requires that two related words conflate to a same stem and two unrelated words to different. This is precisely what Paice's method does. To illustrate the concept further, let us first create a toy stemmer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "lang='fr'\n",
    "\n",
    "with open('lemma/concept_groups_{0}.txt'.format(lang), 'r',encoding='utf8') as file:\n",
    "    concept_groups = [line.rstrip('\\n').split(\",\") for line in file.readlines()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate the metrics  𝐺𝐷𝑀𝑇  and  𝐺𝐷𝑁𝑇  for the concept group loaded above."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GDMT =  1898.0\n",
      "GDNT =  8967832.0\n"
     ]
    }
   ],
   "source": [
    "\n",
    "#concept_groups = concept_groups[0:10]\n",
    "def GDMT(cg):\n",
    "    gdmt = 0\n",
    "    for g in cg:\n",
    "        ng = len(g)\n",
    "        gdmt += 0.5 * ng * (ng-1)\n",
    "    return gdmt\n",
    "\n",
    "def GDNT(cg):\n",
    "    gdnt = 0\n",
    "    W = sum([len(g) for g in cg])\n",
    "    for g in cg:\n",
    "        ng = len(g)\n",
    "        gdnt += 0.5 * ng * (W-ng)\n",
    "    return gdnt\n",
    "\n",
    "print (\"GDMT = \", GDMT(concept_groups))\n",
    "print (\"GDNT = \" , GDNT(concept_groups))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FR nlp Understemming Error = 9.91%\n",
      "FR spicy Understemming Error = 16.28%\n",
      "FR LightFr Understemming Error = 11.33%\n",
      "FR Porter Understemming Error = 3.58%\n",
      "TruncateStem1 Understemming Error = 99.42%\n",
      "TruncateStem2 Understemming Error = 98.63%\n",
      "TruncateStem3 Understemming Error = 98.26%\n",
      "TruncateStem4 Understemming Error = 92.31%\n",
      "TruncateStem5 Understemming Error = 82.56%\n",
      "TruncateStem6 Understemming Error = 67.02%\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "def GUMT(cg, stemmer, debug=False):\n",
    "    gumt = 0\n",
    "    for g in cg:\n",
    "        if debug:\n",
    "            print(\"For group \" , repr(g) )\n",
    "        stems = stemmer.stem_words(g)\n",
    "        ng = len(g)\n",
    "        unique_stems = Counter(stems)\n",
    "        if debug:\n",
    "            print(\"Unique stems with their instances are: \" ,\n",
    "             repr(dict(unique_stems)))\n",
    "        umt = 0\n",
    "        for unique_stem, count in unique_stems.items():\n",
    "            umt += 0.5 * count * (ng-count)\n",
    "        gumt += umt\n",
    "        if debug:\n",
    "            print(\"Unmerged concept pairs in this group = \", umt)\n",
    "\n",
    "    return gumt\n",
    "\n",
    "def UI(cg, stemmer):\n",
    "    return GUMT(cg, stemmer)/GDMT(cg)\n",
    "\n",
    "for stemmer in stemmers:\n",
    "    print(stemmer.name, \"Understemming Error = %.2f%%\" % (UI(concept_groups, stemmer)*100))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Wrongly merged concept pairs and Overstemming Error\n",
    "As discussed earlier, if a non-concept pair is merged together, it is an overstemming error. To count the number of these wrongly merged pairs, a stem group has to be constructed. Where a concept group meant the group of words that should be reduced to a same stem, a stem group is the group of words that actually get reduced to the same stem by the stemmer. For example, consider our concept group from before. It would yield the following stem groups."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "from json import dumps\n",
    "\n",
    "def stem_groups(cg, stm):\n",
    "    \"\"\"\n",
    "    cg is the concept group\n",
    "    stm is the stemmer\n",
    "    \"\"\"\n",
    "    stem_group = defaultdict(list)\n",
    "    stem_group_inverted_cg = defaultdict(list)\n",
    "    for idx, g in enumerate(cg):\n",
    "        for word in g:\n",
    "            stem = stm.stem_word(word)\n",
    "            stem_group[stem].append(word)\n",
    "            #if len(stem_group_inverted_cg[stem])>0 and not str(idx) in stem_group_inverted_cg[stem]:\n",
    "             #   print(\"overstemming case:\",word,stem, ' '.join(str(stem_group_inverted_cg[stem])), str(idx)) \n",
    "            stem_group_inverted_cg[stem].append(str(idx))\n",
    "    return stem_group, stem_group_inverted_cg\n",
    "\n",
    "#print(dumps(stem_groups(concept_groups, porterFr_stemmer), indent=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FR nlp Overstemming Error = 0.0019%\n",
      "FR spicy Overstemming Error = 0.0041%\n",
      "FR LightFr Overstemming Error = 0.0054%\n",
      "FR Porter Overstemming Error = 0.0063%\n",
      "TruncateStem1 Overstemming Error = 0.0031%\n",
      "TruncateStem2 Overstemming Error = 0.0254%\n",
      "TruncateStem3 Overstemming Error = 0.2631%\n",
      "TruncateStem4 Overstemming Error = 1.7744%\n",
      "TruncateStem5 Overstemming Error = 6.3259%\n",
      "TruncateStem6 Overstemming Error = 14.9589%\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "def GWMT(stem_group, stem_group_inverted_cg, debug=False):\n",
    "    gwmt = 0\n",
    "    for sg_key in stem_group.keys():\n",
    "        g = stem_group[sg_key]\n",
    "        if debug:\n",
    "            print(\"For group \" , repr(g) )\n",
    "        ng = len(g)\n",
    "        unique_stems = Counter( stem_group_inverted_cg[sg_key])\n",
    "        if debug:\n",
    "            print(\"Unique cg with their instances are: \" ,\n",
    "             repr(dict(unique_stems)))\n",
    "        wmt = 0\n",
    "        for unique_stem, count in unique_stems.items():\n",
    "            coef = get_word_weight(unique_stem) # weighted coef.\n",
    "            wmt += 0.5 * count * (ng-count) * coef            \n",
    "        gwmt += wmt\n",
    "        if debug:\n",
    "            print(\"Overmerged concept pairs in this group = \", wmt)\n",
    "\n",
    "    return gwmt\n",
    "\n",
    "def WI(cg, stm):\n",
    "    stem_group, stem_group_inverted_cg = stem_groups(cg,stm)\n",
    "    return GWMT(stem_group, stem_group_inverted_cg)/GDNT(cg)\n",
    "for stemmer in stemmers:\n",
    "    print(stemmer.name, \"Overstemming Error = %.4f%%\" % (WI(concept_groups,  stemmer)*100))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}