LDA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\gensim\\utils.py:1212: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n"
     ]
    }
   ],
   "source": [
    "import gensim\n",
    "from gensim.test.utils import common_texts\n",
    "from gensim.corpora.dictionary import Dictionary\n",
    "import nltk\n",
    "import os\n",
    "import re\n",
    "import json\n",
    "import math\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_folder = \"data/NewsArticles/\"\n",
    "files = os.listdir(data_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "files = sorted(files)\n",
    "doc_id = {}\n",
    "for i in range(len(files)):\n",
    "    doc_id['doc'+str(i)] = files[i]\n",
    "with open('data/doc_ids.json','w') as f:\n",
    "    json.dump(doc_id, f, indent=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(\"STOPWORDS\", 'r') as f:\n",
    "    STOPWORDS = set(f.read().split(\"\\n\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "s = set(nltk.corpus.stopwords.words('english')) | STOPWORDS\n",
    "s.add('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'',\n",
       " \"'ll\",\n",
       " 'a',\n",
       " 'able',\n",
       " 'about',\n",
       " 'above',\n",
       " 'abst',\n",
       " 'accordance',\n",
       " 'according',\n",
       " 'accordingly',\n",
       " 'across',\n",
       " 'act',\n",
       " 'actually',\n",
       " 'added',\n",
       " 'adj',\n",
       " 'affected',\n",
       " 'affecting',\n",
       " 'affects',\n",
       " 'after',\n",
       " 'afterwards',\n",
       " 'again',\n",
       " 'against',\n",
       " 'ah',\n",
       " 'ain',\n",
       " 'all',\n",
       " 'almost',\n",
       " 'alone',\n",
       " 'along',\n",
       " 'already',\n",
       " 'also',\n",
       " 'although',\n",
       " 'always',\n",
       " 'am',\n",
       " 'among',\n",
       " 'amongst',\n",
       " 'an',\n",
       " 'and',\n",
       " 'announce',\n",
       " 'another',\n",
       " 'any',\n",
       " 'anybody',\n",
       " 'anyhow',\n",
       " 'anymore',\n",
       " 'anyone',\n",
       " 'anything',\n",
       " 'anyway',\n",
       " 'anyways',\n",
       " 'anywhere',\n",
       " 'apparently',\n",
       " 'approximately',\n",
       " 'are',\n",
       " 'aren',\n",
       " \"aren't\",\n",
       " 'arent',\n",
       " 'arise',\n",
       " 'around',\n",
       " 'as',\n",
       " 'aside',\n",
       " 'ask',\n",
       " 'asking',\n",
       " 'at',\n",
       " 'auth',\n",
       " 'available',\n",
       " 'away',\n",
       " 'awfully',\n",
       " 'b',\n",
       " 'back',\n",
       " 'be',\n",
       " 'became',\n",
       " 'because',\n",
       " 'become',\n",
       " 'becomes',\n",
       " 'becoming',\n",
       " 'been',\n",
       " 'before',\n",
       " 'beforehand',\n",
       " 'begin',\n",
       " 'beginning',\n",
       " 'beginnings',\n",
       " 'begins',\n",
       " 'behind',\n",
       " 'being',\n",
       " 'believe',\n",
       " 'below',\n",
       " 'beside',\n",
       " 'besides',\n",
       " 'between',\n",
       " 'beyond',\n",
       " 'biol',\n",
       " 'both',\n",
       " 'brief',\n",
       " 'briefly',\n",
       " 'but',\n",
       " 'by',\n",
       " 'c',\n",
       " 'ca',\n",
       " 'came',\n",
       " 'can',\n",
       " \"can't\",\n",
       " 'cannot',\n",
       " 'cause',\n",
       " 'causes',\n",
       " 'certain',\n",
       " 'certainly',\n",
       " 'co',\n",
       " 'com',\n",
       " 'come',\n",
       " 'comes',\n",
       " 'contain',\n",
       " 'containing',\n",
       " 'contains',\n",
       " 'could',\n",
       " 'couldn',\n",
       " \"couldn't\",\n",
       " 'couldnt',\n",
       " 'd',\n",
       " 'date',\n",
       " 'did',\n",
       " 'didn',\n",
       " \"didn't\",\n",
       " 'different',\n",
       " 'do',\n",
       " 'does',\n",
       " 'doesn',\n",
       " \"doesn't\",\n",
       " 'doing',\n",
       " 'don',\n",
       " \"don't\",\n",
       " 'done',\n",
       " 'down',\n",
       " 'downwards',\n",
       " 'due',\n",
       " 'during',\n",
       " 'e',\n",
       " 'each',\n",
       " 'ed',\n",
       " 'edu',\n",
       " 'effect',\n",
       " 'eg',\n",
       " 'eight',\n",
       " 'eighty',\n",
       " 'either',\n",
       " 'else',\n",
       " 'elsewhere',\n",
       " 'end',\n",
       " 'ending',\n",
       " 'enough',\n",
       " 'especially',\n",
       " 'et',\n",
       " 'et-al',\n",
       " 'etc',\n",
       " 'even',\n",
       " 'ever',\n",
       " 'every',\n",
       " 'everybody',\n",
       " 'everyone',\n",
       " 'everything',\n",
       " 'everywhere',\n",
       " 'ex',\n",
       " 'except',\n",
       " 'f',\n",
       " 'far',\n",
       " 'few',\n",
       " 'ff',\n",
       " 'fifth',\n",
       " 'first',\n",
       " 'five',\n",
       " 'fix',\n",
       " 'followed',\n",
       " 'following',\n",
       " 'follows',\n",
       " 'for',\n",
       " 'former',\n",
       " 'formerly',\n",
       " 'forth',\n",
       " 'found',\n",
       " 'four',\n",
       " 'from',\n",
       " 'further',\n",
       " 'furthermore',\n",
       " 'g',\n",
       " 'gave',\n",
       " 'get',\n",
       " 'gets',\n",
       " 'getting',\n",
       " 'give',\n",
       " 'given',\n",
       " 'gives',\n",
       " 'giving',\n",
       " 'go',\n",
       " 'goes',\n",
       " 'gone',\n",
       " 'got',\n",
       " 'gotten',\n",
       " 'h',\n",
       " 'had',\n",
       " 'hadn',\n",
       " \"hadn't\",\n",
       " 'happens',\n",
       " 'hardly',\n",
       " 'has',\n",
       " 'hasn',\n",
       " \"hasn't\",\n",
       " 'have',\n",
       " 'haven',\n",
       " \"haven't\",\n",
       " 'having',\n",
       " 'he',\n",
       " 'hed',\n",
       " 'hence',\n",
       " 'her',\n",
       " 'here',\n",
       " 'hereafter',\n",
       " 'hereby',\n",
       " 'herein',\n",
       " 'heres',\n",
       " 'hereupon',\n",
       " 'hers',\n",
       " 'herself',\n",
       " 'hes',\n",
       " 'hi',\n",
       " 'hid',\n",
       " 'him',\n",
       " 'himself',\n",
       " 'his',\n",
       " 'hither',\n",
       " 'home',\n",
       " 'how',\n",
       " 'howbeit',\n",
       " 'however',\n",
       " 'hundred',\n",
       " 'i',\n",
       " \"i'll\",\n",
       " \"i've\",\n",
       " 'id',\n",
       " 'ie',\n",
       " 'if',\n",
       " 'im',\n",
       " 'immediate',\n",
       " 'immediately',\n",
       " 'importance',\n",
       " 'important',\n",
       " 'in',\n",
       " 'inc',\n",
       " 'indeed',\n",
       " 'index',\n",
       " 'information',\n",
       " 'instead',\n",
       " 'into',\n",
       " 'invention',\n",
       " 'inward',\n",
       " 'is',\n",
       " 'isn',\n",
       " \"isn't\",\n",
       " 'it',\n",
       " \"it'll\",\n",
       " \"it's\",\n",
       " 'itd',\n",
       " 'its',\n",
       " 'itself',\n",
       " 'j',\n",
       " 'just',\n",
       " 'k',\n",
       " 'keep \\tkeeps',\n",
       " 'kept',\n",
       " 'kg',\n",
       " 'km',\n",
       " 'know',\n",
       " 'known',\n",
       " 'knows',\n",
       " 'l',\n",
       " 'largely',\n",
       " 'last',\n",
       " 'lately',\n",
       " 'later',\n",
       " 'latter',\n",
       " 'latterly',\n",
       " 'least',\n",
       " 'less',\n",
       " 'lest',\n",
       " 'let',\n",
       " 'lets',\n",
       " 'like',\n",
       " 'liked',\n",
       " 'likely',\n",
       " 'line',\n",
       " 'little',\n",
       " 'll',\n",
       " 'look',\n",
       " 'looking',\n",
       " 'looks',\n",
       " 'ltd',\n",
       " 'm',\n",
       " 'ma',\n",
       " 'made',\n",
       " 'mainly',\n",
       " 'make',\n",
       " 'makes',\n",
       " 'many',\n",
       " 'may',\n",
       " 'maybe',\n",
       " 'me',\n",
       " 'mean',\n",
       " 'means',\n",
       " 'meantime',\n",
       " 'meanwhile',\n",
       " 'merely',\n",
       " 'mg',\n",
       " 'might',\n",
       " 'mightn',\n",
       " \"mightn't\",\n",
       " 'million',\n",
       " 'miss',\n",
       " 'ml',\n",
       " 'more',\n",
       " 'moreover',\n",
       " 'most',\n",
       " 'mostly',\n",
       " 'mr',\n",
       " 'mrs',\n",
       " 'much',\n",
       " 'mug',\n",
       " 'must',\n",
       " 'mustn',\n",
       " \"mustn't\",\n",
       " 'my',\n",
       " 'myself',\n",
       " 'n',\n",
       " 'na',\n",
       " 'name',\n",
       " 'namely',\n",
       " 'nay',\n",
       " 'nd',\n",
       " 'near',\n",
       " 'nearly',\n",
       " 'necessarily',\n",
       " 'necessary',\n",
       " 'need',\n",
       " 'needn',\n",
       " \"needn't\",\n",
       " 'needs',\n",
       " 'neither',\n",
       " 'never',\n",
       " 'nevertheless',\n",
       " 'new',\n",
       " 'next',\n",
       " 'nine',\n",
       " 'ninety',\n",
       " 'no',\n",
       " 'nobody',\n",
       " 'non',\n",
       " 'none',\n",
       " 'nonetheless',\n",
       " 'noone',\n",
       " 'nor',\n",
       " 'normally',\n",
       " 'nos',\n",
       " 'not',\n",
       " 'noted',\n",
       " 'nothing',\n",
       " 'now',\n",
       " 'nowhere',\n",
       " 'o',\n",
       " 'obtain',\n",
       " 'obtained',\n",
       " 'obviously',\n",
       " 'of',\n",
       " 'off',\n",
       " 'often',\n",
       " 'oh',\n",
       " 'ok',\n",
       " 'okay',\n",
       " 'old',\n",
       " 'omitted',\n",
       " 'on',\n",
       " 'once',\n",
       " 'one',\n",
       " 'ones',\n",
       " 'only',\n",
       " 'onto',\n",
       " 'or',\n",
       " 'ord',\n",
       " 'other',\n",
       " 'others',\n",
       " 'otherwise',\n",
       " 'ought',\n",
       " 'our',\n",
       " 'ours',\n",
       " 'ourselves',\n",
       " 'out',\n",
       " 'outside',\n",
       " 'over',\n",
       " 'overall',\n",
       " 'owing',\n",
       " 'own',\n",
       " 'p',\n",
       " 'page',\n",
       " 'pages',\n",
       " 'part',\n",
       " 'particular',\n",
       " 'particularly',\n",
       " 'past',\n",
       " 'per',\n",
       " 'perhaps',\n",
       " 'placed',\n",
       " 'please',\n",
       " 'plus',\n",
       " 'poorly',\n",
       " 'possible',\n",
       " 'possibly',\n",
       " 'potentially',\n",
       " 'pp',\n",
       " 'predominantly',\n",
       " 'present',\n",
       " 'previously',\n",
       " 'primarily',\n",
       " 'probably',\n",
       " 'promptly',\n",
       " 'proud',\n",
       " 'provides',\n",
       " 'put',\n",
       " 'q',\n",
       " 'que',\n",
       " 'quickly',\n",
       " 'quite',\n",
       " 'qv',\n",
       " 'r',\n",
       " 'ran',\n",
       " 'rather',\n",
       " 'rd',\n",
       " 're',\n",
       " 'readily',\n",
       " 'really',\n",
       " 'recent',\n",
       " 'recently',\n",
       " 'ref',\n",
       " 'refs',\n",
       " 'regarding',\n",
       " 'regardless',\n",
       " 'regards',\n",
       " 'related',\n",
       " 'relatively',\n",
       " 'research',\n",
       " 'respectively',\n",
       " 'resulted',\n",
       " 'resulting',\n",
       " 'results',\n",
       " 'right',\n",
       " 'run',\n",
       " 's',\n",
       " 'said',\n",
       " 'same',\n",
       " 'saw',\n",
       " 'say',\n",
       " 'saying',\n",
       " 'says',\n",
       " 'sec',\n",
       " 'section',\n",
       " 'see',\n",
       " 'seeing',\n",
       " 'seem',\n",
       " 'seemed',\n",
       " 'seeming',\n",
       " 'seems',\n",
       " 'seen',\n",
       " 'self',\n",
       " 'selves',\n",
       " 'sent',\n",
       " 'seven',\n",
       " 'several',\n",
       " 'shall',\n",
       " 'shan',\n",
       " \"shan't\",\n",
       " 'she',\n",
       " \"she'll\",\n",
       " \"she's\",\n",
       " 'shed',\n",
       " 'shes',\n",
       " 'should',\n",
       " \"should've\",\n",
       " 'shouldn',\n",
       " \"shouldn't\",\n",
       " 'show',\n",
       " 'showed',\n",
       " 'shown',\n",
       " 'showns',\n",
       " 'shows',\n",
       " 'significant',\n",
       " 'significantly',\n",
       " 'similar',\n",
       " 'similarly',\n",
       " 'since',\n",
       " 'six',\n",
       " 'slightly',\n",
       " 'so',\n",
       " 'some',\n",
       " 'somebody',\n",
       " 'somehow',\n",
       " 'someone',\n",
       " 'somethan',\n",
       " 'something',\n",
       " 'sometime',\n",
       " 'sometimes',\n",
       " 'somewhat',\n",
       " 'somewhere',\n",
       " 'soon',\n",
       " 'sorry',\n",
       " 'specifically',\n",
       " 'specified',\n",
       " 'specify',\n",
       " 'specifying',\n",
       " 'still',\n",
       " 'stop',\n",
       " 'strongly',\n",
       " 'sub',\n",
       " 'substantially',\n",
       " 'successfully',\n",
       " 'such',\n",
       " 'sufficiently',\n",
       " 'suggest',\n",
       " 'sup',\n",
       " 'sure',\n",
       " 't',\n",
       " 'than',\n",
       " 'that',\n",
       " \"that'll\",\n",
       " 'the',\n",
       " 'their',\n",
       " 'theirs',\n",
       " 'them',\n",
       " 'themselves',\n",
       " 'then',\n",
       " 'there',\n",
       " 'these',\n",
       " 'they',\n",
       " 'this',\n",
       " 'those',\n",
       " 'through',\n",
       " 'to',\n",
       " 'too',\n",
       " 'under',\n",
       " 'until',\n",
       " 'up',\n",
       " 've',\n",
       " 'very',\n",
       " 'was',\n",
       " 'wasn',\n",
       " \"wasn't\",\n",
       " 'we',\n",
       " 'were',\n",
       " 'weren',\n",
       " \"weren't\",\n",
       " 'what',\n",
       " 'when',\n",
       " 'where',\n",
       " 'which',\n",
       " 'while',\n",
       " 'who',\n",
       " 'whom',\n",
       " 'why',\n",
       " 'will',\n",
       " 'with',\n",
       " 'won',\n",
       " \"won't\",\n",
       " 'wouldn',\n",
       " \"wouldn't\",\n",
       " 'y',\n",
       " 'you',\n",
       " \"you'd\",\n",
       " \"you'll\",\n",
       " \"you're\",\n",
       " \"you've\",\n",
       " 'your',\n",
       " 'yours',\n",
       " 'yourself',\n",
       " 'yourselves'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "texts = []\n",
    "text_dict = {}\n",
    "for file in files:\n",
    "    with open(data_folder+file,encoding=\"ISO-8859-1\") as f:\n",
    "        text = f.read()\n",
    "        text = re.split('[, \\.\\n]', text)\n",
    "        text = [word.lower() for word in text if word.lower() not in s and word.isalpha() and len(word)>=3]\n",
    "        texts.append(text)\n",
    "        text_dict[file] = text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "common_dictionary = Dictionary(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "common_corpus = [common_dictionary.doc2bow(text) for text in texts]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lda = gensim.models.LdaModel(common_corpus,id2word=common_dictionary ,num_topics=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(14, 0.9876068)]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda.get_document_topics(common_dictionary.doc2bow(texts[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Document Topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "doc_topic_json = {}\n",
    "for i in doc_id:\n",
    "    topic_array = [0 for i in range(30)]\n",
    "    topics = lda.get_document_topics(common_dictionary.doc2bow(text_dict[doc_id[i]]))\n",
    "    for tpc in topics:\n",
    "        topic_array[tpc[0]] = str(tpc[1])\n",
    "    doc_topic_json[i] = topic_array\n",
    "with open(\"data/doc_topic.json\",\"w\") as f:\n",
    "    json.dump(doc_topic_json, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text v/s Topic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "matrix = [[str(0) for i in range(150)] for i in range(len(texts))]\n",
    "\n",
    "### Increase for more Radius\n",
    "factor = 1\n",
    "###\n",
    "\n",
    "for i in range(len(texts)):\n",
    "    topics = lda.get_document_topics(common_dictionary.doc2bow(texts[i]))\n",
    "    for t in topics:\n",
    "        matrix[i][t[0]] = str(t[1] * factor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "string = ','+','.join([\"Topic\"+str(i) for i in range(30)])\n",
    "for i in range(len(texts)):\n",
    "    string += '\\ndoc' + str(i) + ',' + ','.join(matrix[i])\n",
    "\n",
    "with open(\"data/text_topic_matrix.csv\", \"w\") as f:\n",
    "    f.write(string)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word v/s Topic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# csv_string = 'word,value'\n",
    "\n",
    "# for t in lda.get_topic_terms(1, topn=100):\n",
    "# #     print( lda.id2word[t[0]],t[1],t[0])\n",
    "#     csv_string += '\\n' + lda.id2word[t[0]] + ',' + str(t[1])\n",
    "# # t = lda.get_topic_terms(57, topn=100)\n",
    "\n",
    "# with open('topic1.csv', 'w') as f:\n",
    "#     f.write(csv_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1589"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = {}\n",
    "for k,v in lda.id2word.items():\n",
    "    words[v] = k\n",
    "with open(\"data/word_ids.json\", \"w\") as f:\n",
    "    json.dump(words, f)\n",
    "len(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word_dict = {}\n",
    "for topic in range(30):\n",
    "    word_list = lda.get_topic_terms(topic, topn=100)\n",
    "    for word_tuple in word_list:\n",
    "        word_id = word_tuple[0]\n",
    "        word_score = word_tuple[1]\n",
    "        if str(word_id) not in word_dict:\n",
    "            word_dict[str(word_id)] = {}\n",
    "        word_dict[str(word_id)][topic] = str(word_score)\n",
    "\n",
    "word_dict\n",
    "with open(\"data/word_topic.json\", \"w\") as f:\n",
    "    json.dump(word_dict, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DOC TEXT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n"
     ]
    }
   ],
   "source": [
    "texts = []\n",
    "ix = 0\n",
    "for file in files:\n",
    "    print(ix)\n",
    "    with open(data_folder+file,encoding=\"ISO-8859-1\") as f:\n",
    "        text = \"\\n\"+f.read()\n",
    "        text_words = set(re.split('[, \\.\\n]', text))\n",
    "        html_string = text\n",
    "        for word in text_words:\n",
    "            if word.lower() in words and str(words[word.lower()]) in word_dict:\n",
    "                keys = list(word_dict[str(words[word.lower()])].keys())\n",
    "                classes = ''\n",
    "                for i in keys:\n",
    "                    classes += 't'+str(i)+\" \"\n",
    "#                 html_string = html_string.replace(r\"\\b\"+word+\"\\b\",\"<span class=\\\"\"+classes+\"hoverable\\\">\"+word+\"</span>\")\n",
    "                html_string = html_string.replace(\"\\n\"+word+\" \",\"\\n<span class=\\\"\"+classes+\"hoverable\\\">\"+word+\"</span> \")\n",
    "                html_string = html_string.replace(\" \"+word+\"\\n\",\" <span class=\\\"\"+classes+\"hoverable\\\">\"+word+\"</span>\\n\")\n",
    "                html_string = html_string.replace(\" \"+word+\" \",\" <span class=\\\"\"+classes+\"hoverable\\\">\"+word+\"</span> \")\n",
    "                html_string = html_string.replace(\"\\n\"+word+\"\\n\",\"\\n<span class=\\\"\"+classes+\"hoverable\\\">\"+word+\"</span>\\n\")\n",
    "#                 html_string = re.sub(word+\" \",\"<span class=\\\"\"+classes+\"hoverable\\\">\"+word+\"</span>\",html_string)\n",
    "#                 html_string = re.sub(\" \"+word,\"<span class=\\\"\"+classes+\"hoverable\\\">\"+word+\"</span>\",html_string)\n",
    "        with open(\"data/doc\"+str(ix)+\".txt\",\"w\",encoding=\"ISO-8859-1\") as f1:\n",
    "            f1.write(html_string)\n",
    "    ix+=1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TOPIC V/S WORDS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for topic in range(30):\n",
    "    word_list = lda.get_topic_terms(topic, topn=100)\n",
    "    file_str = \"word,value\"\n",
    "    for word_tuple in word_list:\n",
    "        word_id = word_tuple[0]\n",
    "        file_str += \"\\n\" + str(lda.id2word[word_tuple[0]]) +\",\"+ str(word_tuple[1])\n",
    "    with open(\"data/Topic\"+str(topic)+\".csv\", \"w\") as f:\n",
    "        f.write(file_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n"
     ]
    }
   ],
   "source": [
    "texts = []\n",
    "ix = 0\n",
    "for file in files:\n",
    "    print(ix)\n",
    "    with open(data_folder+file,encoding=\"ISO-8859-1\") as f:\n",
    "        text = \"\\n\"+f.read()\n",
    "#         lines = text.split(\"\\n\")\n",
    "        line_word_list = re.split('[, \\.\\n]', text)\n",
    "        no_lines = len(line_word_list)//10\n",
    "        \n",
    "        line_graph_data = []\n",
    "        for i in range(30):\n",
    "            l = {\n",
    "                \"name\": \"Topic\"+str(i),\n",
    "                \"id\": \"t\"+str(i),\n",
    "                \"values\": []\n",
    "            }\n",
    "            line_graph_data.append(l)\n",
    "        \n",
    "        \n",
    "        for line_index in range(no_lines):\n",
    "            line_words = line_word_list[10*line_index:10*(line_index+1)]\n",
    "            line_scores = {}\n",
    "            for word in line_words:\n",
    "                if word.lower() in words and str(words[word.lower()]) in word_dict:\n",
    "                    t_scores = word_dict[str(words[word.lower()])]\n",
    "                    for t_ix in t_scores:\n",
    "                        word_score = float(t_scores[t_ix]) \n",
    "                        if t_ix not in line_scores:\n",
    "                            line_scores[t_ix] = 0\n",
    "                        line_scores[t_ix] += word_score\n",
    "\n",
    "            for topic in range(30):\n",
    "                if topic in line_scores:\n",
    "                    line_graph_data[topic][\"values\"].append({\"line\":str(line_index+1), \"frequency\":str(line_scores[topic])})\n",
    "                else:\n",
    "                    line_graph_data[topic][\"values\"].append({\"line\":str(line_index+1), \"frequency\":str(0)})\n",
    "    with open(\"data/doc\"+str(ix)+\"_linegraph.json\", \"w\") as f:\n",
    "        json.dump({\"data\":line_graph_data}, f, indent=True)\n",
    "    ix+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}