gcj_baseline_train-py.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pickle\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# some params\n",
    "\n",
    "MIN_FILE_WORDS = 50\n",
    "MIN_SUBMISSIONS_PER_USER = 10\n",
    "NUMBER_OF_USERS = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# download training dataset: https://github.com/Jur1cek/gcj-dataset/raw/master/gcj2020.csv.tar.bz2 do not forget to unbzip2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/gcj2020.csv', dtype=str)\n",
    "\n",
    "# Just Python 3\n",
    "df_count = df.loc[df['full_path'].str.match(\".*\\.PYTHON3$\", case=False)]\n",
    "df_count = df_count.drop_duplicates(subset='flines')\n",
    "\n",
    "# minimum of 50 WORDS\n",
    "df_count['counts'] = [len(str(x).split()) for x in df_count['flines']]\n",
    "df_count = df_count.loc[df_count['counts'] >= MIN_FILE_WORDS] \n",
    "\n",
    "# minimum of 10 submissions\n",
    "df_count = df_count.groupby('username').filter(lambda x: x['username'].count() >= MIN_SUBMISSIONS_PER_USER)\n",
    "\n",
    "# random 10 submissions per user\n",
    "df_count_all = df_count.groupby('username').apply(lambda x: x.sample(MIN_SUBMISSIONS_PER_USER))\n",
    "\n",
    "\n",
    "# pick random NUMBER_OF_USERS users\n",
    "df_count_all=df_count_all.reset_index(drop=True)\n",
    "g=df_count_all.groupby('username')\n",
    "a=np.arange(g.ngroups)\n",
    "np.random.shuffle(a)\n",
    "df_count_all=df_count_all[g.ngroup().isin(a[:NUMBER_OF_USERS])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.asarray(df_count_all['flines'].tolist())\n",
    "y = np.asarray(df_count_all['username'].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, shuffle=True, stratify=y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "# be careful of TF-IDF memory usage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer(max_features=5000, strip_accents='unicode', use_idf=True, lowercase=True, ngram_range=(1,4))\n",
    "train_vectors = vectorizer.fit_transform(X_train)\n",
    "test_vectors = vectorizer.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = RandomForestClassifier(n_estimators=100)\n",
    "\n",
    "model.fit(train_vectors, y_train)\n",
    "\n",
    "pred = model.predict(test_vectors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.94"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sklearn.metrics.accuracy_score(y_test, pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['007brendan@gmail.com', 'AQForty', 'AndrewMathias', 'Ashwa462',\n",
       "       'Aurel', 'Ayush1290', 'BargaviC', 'Beurtschipper', 'Bilisel',\n",
       "       'C_BK', 'CoCoMong', 'Emily1616', 'HotCurry', 'HydraulicSheep',\n",
       "       'Jacky_Zheng', 'Jahnavi_M', 'Jeck5ivk', 'Kandhum', 'MakiY',\n",
       "       'Mammouth', 'Marc974', 'MasterBlaster', 'Maw1a_', 'Mitsuo',\n",
       "       'MizhaelZeyuChen', \"Mo'menMohHamdy\", 'Monir786', 'Morne',\n",
       "       'Nandini2000', 'NeutrinoAnt', 'NielsW', 'Omkar3103', 'Poneface',\n",
       "       'Rohit1905', 'RutujaRane', 'Saketkr21', 'Seb314', 'StormSpirit',\n",
       "       'Tecnosam', 'TheBoaredProgrammer', 'Tr3cks', 'Volifter',\n",
       "       'ZulqarnainAbbas', 'ZytchiX', 'abhirao', 'achyutha',\n",
       "       'alvinj.delacruz@gmail.com', 'bHaSkAr', 'benguin', 'carlossless',\n",
       "       'cillinzhang', 'coffeered', 'daVinci', 'deep.sarkar', 'dias17',\n",
       "       'dorime', 'drakeire', 'erencantemur', 'georgpap@isi.edu',\n",
       "       'ishikashah2510', 'jbrummer402', 'jedrzejmikolajczyk',\n",
       "       'jeholliday', 'joaobernardo01', 'kappahouse', 'kawasaki',\n",
       "       'kiritowo', 'lazylife', 'loke26', 'louis.renaux@berkeley.edu',\n",
       "       'machy', 'markopuza', 'metanovitia', 'mickie895', 'mortal_1',\n",
       "       'mustafahaddara', 'nandangrover', 'nikhil360', 'rhfktj', 'ricsi98',\n",
       "       'sai_ganesh', 'salaschen', 'samlushadi', 'sanDeep', 'shivamsharma',\n",
       "       'simha0204', 'simpleapples', 'sourcestrong', 'spence', 'szalaib',\n",
       "       'the-maze-runner', 'thusharakart', 'vatsal-kandoi',\n",
       "       'vladomar@gmail.com', 'vutung2311', 'warrior123',\n",
       "       'waterstorm64@gmail.com', 'x1Mike7x', 'yingying2019', 'zachbodi'],\n",
       "      dtype='<U25')"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# complete list of authors\n",
    "np.unique(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*Author:* Seb314\n",
      "t=int(input())\n",
      "soluce=\"\"\n",
      "\n",
      "for i in range (0,t):\n",
      "  line = input()\n",
      "  start_x=int(line.split(\" \")[0])\n",
      "  start_y=int(line.split(\" \")[1])\n",
      "  new_y = start_y\n",
      "  moves = list(line.split(\" \")[2])\n",
      "  soluce = \"\"\n",
      "\n",
      "  #aller au bon X\n",
      "  for k in range (0,start_x):\n",
      "    if len(moves) > 0:\n",
      "      move = moves.pop(0)\n",
      "      if move == \"N\":\n",
      "        new_y = new_y + 1\n",
      "      else:\n",
      "        new_y = new_y - 1\n",
      "    else:\n",
      "      soluce = \"IMPOSSIBLE\"\n",
      "      break\n",
      "  \n",
      "  if soluce == \"IMPOSSIBLE\":\n",
      "    a=1\n",
      "  else:\n",
      "    if new_y == 0 :\n",
      "      soluce = start_x\n",
      "    else:\n",
      "      for j in range(0,len(moves)):\n",
      "        move = moves[j]\n",
      "        if move == \"N\":\n",
      "          new_y = new_y + 1\n",
      "        else:\n",
      "          new_y = new_y - 1\n",
      "        \n",
      "        if j >= abs(new_y):\n",
      "          soluce = str(start_x + j+1)\n",
      "          break\n",
      "\n",
      "  if soluce == \"\":\n",
      "    soluce = \"IMPOSSIBLE\"\n",
      "\n",
      "  print(\"Case #\" + str(i+1) + \": \" + str(soluce))\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Train print:\n",
    "print(\"*Author:*\", y_train[0])\n",
    "print(X_train[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*Author:* x1Mike7x\n",
      "DEBUG = 0\n",
      "\n",
      "\n",
      "def print_pascal(n=10):\n",
      "    p = [[1], [1, 1]]\n",
      "    for i in range(2, n + 1):\n",
      "        a = [1]\n",
      "        for j in range(1, i + 1):\n",
      "            k = p[-1][j - 1]\n",
      "            if j < len(p[-1]):\n",
      "                k += p[-1][j]\n",
      "            a.append(k)\n",
      "        p.append(a)\n",
      "    for line in p:\n",
      "        print(str(sum(line)), '  |  ', ' '.join(map(str, line)))\n",
      "\n",
      "\n",
      "def get_tests(use_file):\n",
      "    if use_file:\n",
      "        tests = []\n",
      "        with open('test.txt', 'r') as ifs:\n",
      "            lines = ifs.readlines()\n",
      "            n = int(lines[0])\n",
      "            return list(map(int, lines[1:n+1]))\n",
      "    else:\n",
      "        T = int(input())\n",
      "        tests = []\n",
      "        for _ in range(T):\n",
      "            n = int(input())\n",
      "            tests.append(n)\n",
      "    return tests\n",
      "\n",
      "\n",
      "def solution(n):\n",
      "    K = 42\n",
      "    if n < K + 5:\n",
      "        res = [(i, 1) for i in range(1, n + 1)]\n",
      "        return '\\n'.join('%d %d' % (r, k) for r, k in res)\n",
      "    \n",
      "    x = n - K\n",
      "    v = []\n",
      "    b = 1\n",
      "    while x > 0:\n",
      "        if x & 1:\n",
      "            v.append(b)\n",
      "        x >>= 1\n",
      "        b += 1\n",
      "    \n",
      "    res = []\n",
      "    s = 0\n",
      "    i = 0\n",
      "    row = 1\n",
      "    should_rev = False\n",
      "    while s < n:\n",
      "        if i >= len(v) or v[i] != row:\n",
      "            if should_rev:\n",
      "                res.append((row, row))\n",
      "            else:\n",
      "                res.append((row, 1))\n",
      "            s += 1\n",
      "        else:\n",
      "            z = []\n",
      "            for j in range(row):\n",
      "                z.append((row, j + 1))\n",
      "            s += 1 << (row - 1)\n",
      "            if should_rev:\n",
      "                res.extend(z[::-1])\n",
      "            else:\n",
      "                res.extend(z)\n",
      "            should_rev = not should_rev\n",
      "            i += 1\n",
      "        row += 1\n",
      "    return '\\n'.join('%d %d' % (r, k) for r, k in res)\n",
      "    \n",
      "\n",
      "def main(is_debug):\n",
      "    tests = get_tests(is_debug)\n",
      "    for test, n in enumerate(tests):\n",
      "        result = solution(n)\n",
      "        print('Case #%d:\\n%s' % (test + 1, result))\n",
      "\n",
      "\n",
      "if __name__ == '__main__':\n",
      "    main(DEBUG)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Test print:\n",
    "print(\"*Author:*\", y_test[0])\n",
    "print(X_test[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save\n",
    "\n",
    "with open('model.pickle', 'wb') as handle:\n",
    "    pickle.dump(model, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    \n",
    "with open('vectorizer.pickle', 'wb') as handle:\n",
    "    pickle.dump(vectorizer, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "# X_train, X_test, y_train, y_test\n",
    "with open('X_train.pickle', 'wb') as handle:\n",
    "    pickle.dump(X_train, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    \n",
    "with open('X_test.pickle', 'wb') as handle:\n",
    "    pickle.dump(X_test, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "with open('y_train.pickle', 'wb') as handle:\n",
    "    pickle.dump(y_train, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    \n",
    "with open('y_test.pickle', 'wb') as handle:\n",
    "    pickle.dump(y_test, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}