[Notebook2]Feature_Engineering_and_Encoding.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import libraries \n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Both comp_bundas_train and comp_bundas_train are datasets generated by the code written in R. \n",
    "\n",
    "These datasets have been modified in that the rows with missing data have either had their missing data filled from items with similar Item_IDs or have been removed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "home_folder = '/Users/anthonymiyoro/Documents/code/DataTho/data'\n",
    "\n",
    "comp_bundas_train = pd.read_csv(home_folder + 'comp_bundas_train.csv')\n",
    "test_bundas = pd.read_csv(home_folder + 'comp_bundas_test.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encode Categorical Variables on Test Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4055 entries, 0 to 4054\n",
      "Data columns (total 11 columns):\n",
      "Item_ID                     4055 non-null object\n",
      "Weight                      4055 non-null float64\n",
      "FatContent                  4055 non-null object\n",
      "Visibility                  4055 non-null float64\n",
      "Category                    4055 non-null object\n",
      "Max_Price                   4055 non-null float64\n",
      "Store_ID                    4055 non-null object\n",
      "Store_Establishment_Year    4055 non-null int64\n",
      "Store_Size                  4055 non-null object\n",
      "Store_Location_Type         4055 non-null object\n",
      "Store_Type                  4055 non-null object\n",
      "dtypes: float64(3), int64(1), object(7)\n",
      "memory usage: 348.6+ KB\n"
     ]
    }
   ],
   "source": [
    "test_bundas.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# One hot encode the categorical columns\n",
    "FatContent_one_hot = pd.get_dummies(test_bundas['FatContent'], \n",
    "                                     prefix='FatContent')\n",
    "\n",
    "Category_one_hot = pd.get_dummies(test_bundas['Category'], \n",
    "                                prefix='Category')\n",
    "\n",
    "Store_Size_one_hot = pd.get_dummies(test_bundas['Store_Size'], \n",
    "                                   prefix='Store_Size')\n",
    "\n",
    "# One hot encode the categorical columns\n",
    "Store_Location_Type_one_hot = pd.get_dummies(test_bundas['Store_Location_Type'], \n",
    "                                     prefix='Store_Location_Type')\n",
    "\n",
    "Store_Type_one_hot = pd.get_dummies(test_bundas['Store_Type'], \n",
    "                                prefix='Store_Type')\n",
    "\n",
    "Store_ID_one_hot = pd.get_dummies(test_bundas['Store_ID'], \n",
    "                                   prefix='Store_ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Combine the one hot encoded columns with df_con_enc\n",
    "modDfenc = pd.concat([test_bundas, \n",
    "                        FatContent_one_hot, \n",
    "                        Category_one_hot, \n",
    "                        Store_Size_one_hot,\n",
    "                        Store_Location_Type_one_hot, \n",
    "                        Store_Type_one_hot, \n",
    "                        Store_ID_one_hot], axis=1)\n",
    "\n",
    "# Drop the original categorical columns (because now they've been one hot encoded)\n",
    "modDfenc = modDfenc.drop(['FatContent','Category','Store_Size','Store_Location_Type','Store_Type','Store_ID'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Item_ID</th>\n",
       "      <th>Weight</th>\n",
       "      <th>Visibility</th>\n",
       "      <th>Max_Price</th>\n",
       "      <th>Store_Establishment_Year</th>\n",
       "      <th>FatContent_LF</th>\n",
       "      <th>FatContent_Low Fat</th>\n",
       "      <th>FatContent_Regular</th>\n",
       "      <th>FatContent_low fat</th>\n",
       "      <th>FatContent_reg</th>\n",
       "      <th>...</th>\n",
       "      <th>Store_Type_Supermarket Type1</th>\n",
       "      <th>Store_Type_Supermarket Type2</th>\n",
       "      <th>Store_Type_Supermarket Type3</th>\n",
       "      <th>Store_ID_OUT013</th>\n",
       "      <th>Store_ID_OUT018</th>\n",
       "      <th>Store_ID_OUT019</th>\n",
       "      <th>Store_ID_OUT027</th>\n",
       "      <th>Store_ID_OUT035</th>\n",
       "      <th>Store_ID_OUT046</th>\n",
       "      <th>Store_ID_OUT049</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>FDW58</td>\n",
       "      <td>20.750</td>\n",
       "      <td>0.007565</td>\n",
       "      <td>107.8622</td>\n",
       "      <td>1999</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>FDH56</td>\n",
       "      <td>9.800</td>\n",
       "      <td>0.063817</td>\n",
       "      <td>117.1492</td>\n",
       "      <td>1997</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>FDL48</td>\n",
       "      <td>19.350</td>\n",
       "      <td>0.082602</td>\n",
       "      <td>50.1034</td>\n",
       "      <td>2009</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>FDU11</td>\n",
       "      <td>4.785</td>\n",
       "      <td>0.092738</td>\n",
       "      <td>122.3098</td>\n",
       "      <td>1999</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>DRL59</td>\n",
       "      <td>16.750</td>\n",
       "      <td>0.021206</td>\n",
       "      <td>52.0298</td>\n",
       "      <td>1987</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 43 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  Item_ID  Weight  Visibility  Max_Price  Store_Establishment_Year  \\\n",
       "0   FDW58  20.750    0.007565   107.8622                      1999   \n",
       "1   FDH56   9.800    0.063817   117.1492                      1997   \n",
       "2   FDL48  19.350    0.082602    50.1034                      2009   \n",
       "3   FDU11   4.785    0.092738   122.3098                      1999   \n",
       "4   DRL59  16.750    0.021206    52.0298                      1987   \n",
       "\n",
       "   FatContent_LF  FatContent_Low Fat  FatContent_Regular  FatContent_low fat  \\\n",
       "0              0                   1                   0                   0   \n",
       "1              0                   0                   1                   0   \n",
       "2              0                   0                   1                   0   \n",
       "3              0                   1                   0                   0   \n",
       "4              1                   0                   0                   0   \n",
       "\n",
       "   FatContent_reg  ...  Store_Type_Supermarket Type1  \\\n",
       "0               0  ...                             1   \n",
       "1               0  ...                             1   \n",
       "2               0  ...                             0   \n",
       "3               0  ...                             1   \n",
       "4               0  ...                             1   \n",
       "\n",
       "   Store_Type_Supermarket Type2  Store_Type_Supermarket Type3  \\\n",
       "0                             0                             0   \n",
       "1                             0                             0   \n",
       "2                             1                             0   \n",
       "3                             0                             0   \n",
       "4                             0                             0   \n",
       "\n",
       "   Store_ID_OUT013  Store_ID_OUT018  Store_ID_OUT019  Store_ID_OUT027  \\\n",
       "0                0                0                0                0   \n",
       "1                0                0                0                0   \n",
       "2                0                1                0                0   \n",
       "3                0                0                0                0   \n",
       "4                1                0                0                0   \n",
       "\n",
       "   Store_ID_OUT035  Store_ID_OUT046  Store_ID_OUT049  \n",
       "0                0                0                1  \n",
       "1                0                1                0  \n",
       "2                0                0                0  \n",
       "3                0                0                1  \n",
       "4                0                0                0  \n",
       "\n",
       "[5 rows x 43 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "modDfenc.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4055 entries, 0 to 4054\n",
      "Data columns (total 43 columns):\n",
      "Item_ID                           4055 non-null object\n",
      "Weight                            4055 non-null float64\n",
      "Visibility                        4055 non-null float64\n",
      "Max_Price                         4055 non-null float64\n",
      "Store_Establishment_Year          4055 non-null int64\n",
      "FatContent_LF                     4055 non-null uint8\n",
      "FatContent_Low Fat                4055 non-null uint8\n",
      "FatContent_Regular                4055 non-null uint8\n",
      "FatContent_low fat                4055 non-null uint8\n",
      "FatContent_reg                    4055 non-null uint8\n",
      "Category_Baking Goods             4055 non-null uint8\n",
      "Category_Breads                   4055 non-null uint8\n",
      "Category_Breakfast                4055 non-null uint8\n",
      "Category_Canned                   4055 non-null uint8\n",
      "Category_Dairy                    4055 non-null uint8\n",
      "Category_Frozen Foods             4055 non-null uint8\n",
      "Category_Fruits and Vegetables    4055 non-null uint8\n",
      "Category_Hard Drinks              4055 non-null uint8\n",
      "Category_Health and Hygiene       4055 non-null uint8\n",
      "Category_Household                4055 non-null uint8\n",
      "Category_Meat                     4055 non-null uint8\n",
      "Category_Others                   4055 non-null uint8\n",
      "Category_Seafood                  4055 non-null uint8\n",
      "Category_Snack Foods              4055 non-null uint8\n",
      "Category_Soft Drinks              4055 non-null uint8\n",
      "Category_Starchy Foods            4055 non-null uint8\n",
      "Store_Size_High                   4055 non-null uint8\n",
      "Store_Size_Medium                 4055 non-null uint8\n",
      "Store_Size_Small                  4055 non-null uint8\n",
      "Store_Location_Type_Tier 1        4055 non-null uint8\n",
      "Store_Location_Type_Tier 2        4055 non-null uint8\n",
      "Store_Location_Type_Tier 3        4055 non-null uint8\n",
      "Store_Type_Grocery Store          4055 non-null uint8\n",
      "Store_Type_Supermarket Type1      4055 non-null uint8\n",
      "Store_Type_Supermarket Type2      4055 non-null uint8\n",
      "Store_Type_Supermarket Type3      4055 non-null uint8\n",
      "Store_ID_OUT013                   4055 non-null uint8\n",
      "Store_ID_OUT018                   4055 non-null uint8\n",
      "Store_ID_OUT019                   4055 non-null uint8\n",
      "Store_ID_OUT027                   4055 non-null uint8\n",
      "Store_ID_OUT035                   4055 non-null uint8\n",
      "Store_ID_OUT046                   4055 non-null uint8\n",
      "Store_ID_OUT049                   4055 non-null uint8\n",
      "dtypes: float64(3), int64(1), object(1), uint8(38)\n",
      "memory usage: 309.0+ KB\n"
     ]
    }
   ],
   "source": [
    "modDfenc.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "modDfenc.to_csv(home_folder + 'test_modified.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encode Categorical Variables on Train Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# One hot encode the categorical columns\n",
    "FatContent_one_hot = pd.get_dummies(comp_bundas_train['FatContent'], \n",
    "                                     prefix='FatContent')\n",
    "\n",
    "Category_one_hot = pd.get_dummies(comp_bundas_train['Category'], \n",
    "                                prefix='Category')\n",
    "\n",
    "Store_Size_one_hot = pd.get_dummies(comp_bundas_train['Store_Size'], \n",
    "                                   prefix='Store_Size')\n",
    "\n",
    "# One hot encode the categorical columns\n",
    "Store_Location_Type_one_hot = pd.get_dummies(comp_bundas_train['Store_Location_Type'], \n",
    "                                     prefix='Store_Location_Type')\n",
    "\n",
    "Store_Type_one_hot = pd.get_dummies(comp_bundas_train['Store_Type'], \n",
    "                                prefix='Store_Type')\n",
    "\n",
    "Store_ID_one_hot = pd.get_dummies(comp_bundas_train['Store_ID'], \n",
    "                                   prefix='Store_ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Combine the one hot encoded columns with df_con_enc\n",
    "modDfenc = pd.concat([comp_bundas_train, \n",
    "                        FatContent_one_hot, \n",
    "                        Category_one_hot, \n",
    "                        Store_Size_one_hot,\n",
    "                        Store_Location_Type_one_hot, \n",
    "                        Store_Type_one_hot, \n",
    "                        Store_ID_one_hot], axis=1)\n",
    "\n",
    "# Drop the original categorical columns (because now they've been one hot encoded)\n",
    "modDfenc = modDfenc.drop(['FatContent','Category','Store_Size','Store_Location_Type','Store_Type','Store_ID'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "modDfenc.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "modDfenc.to_csv(home_folder + 'train_modified.csv',index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}