when_to_show/predict_accept_user_study.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from requests import session\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import json\n",
    "import copy\n",
    "import numpy as np, scipy.stats as st\n",
    "import json\n",
    "from tqdm import tqdm\n",
    "import pickle\n",
    "import logging\n",
    "import  argparse\n",
    "import math\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import f1_score\n",
    "from itertools import chain\n",
    "import xgboost as xg\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_error\n",
    "logging.basicConfig(level=logging.INFO)\n",
    "from sklearn import preprocessing\n",
    "import matplotlib\n",
    "from re import S\n",
    "from scipy.stats.stats import pearsonr\n",
    "from sklearn.calibration import calibration_curve\n",
    "import matplotlib\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_features_labels_user_study(path_df, features_to_keep, label_index, REMOVE_S_AND_R, predicting_time = False):\n",
    "    df_observations, feature_dict= pickle.load(open(path_df, 'rb'))\n",
    "    label_to_idx = {'Accepted': 0, 'Rejected': 1, 'Browsing': 2, 'Replay': 3, 'Shown': 4}\n",
    "    idx_to_label = {0: 'Accepted', 1: 'Rejected', 2: 'Browsing', 3: 'Replay', 4: 'Shown'}\n",
    "    # COMMENT THE LINE BELOW IF NOT USING USER STUDY DATA\n",
    "    df_observations = [df_observations]\n",
    "    df_observations_features = []\n",
    "    df_observations_labels = []\n",
    "\n",
    "    for i in tqdm(range(len(df_observations))):\n",
    "        df_user = []\n",
    "        df_user_labels = []\n",
    "        for j in range(len(df_observations[i])):\n",
    "            session_df = []\n",
    "            session_labels = []\n",
    "            for h in range(len(df_observations[i][j])):\n",
    "                features_list = np.array(df_observations[i][j][h])[features_to_keep]\n",
    "                # merge features\n",
    "                features = []\n",
    "                for f in features_list:\n",
    "                    features.extend(f)\n",
    "                features = np.array(features)\n",
    "                if not predicting_time:\n",
    "                    label = label_to_idx[np.array(df_observations[i][j][h])[label_index[0]]]\n",
    "                else:\n",
    "                    label = np.array(df_observations[i][j][h])[label_index[0]]\n",
    "                if REMOVE_S_AND_R and label in [2,3,4]:\n",
    "                    continue\n",
    "                session_df.append(features)\n",
    "                session_labels.append(label)\n",
    "            if len(session_df) > 0:\n",
    "                df_user.append(session_df)\n",
    "                df_user_labels.append(session_labels)\n",
    "        if len(df_user) > 0:\n",
    "            df_observations_features.append(df_user)\n",
    "            df_observations_labels.append(df_user_labels)\n",
    "        # save space and delete df_observations[i]\n",
    "        df_observations[i] = None\n",
    "    df_observations_features =  df_observations_features[0]\n",
    "    # make each element a list\n",
    "    df_observations_features = [ [df_observations_features[i]] for i in range(len(df_observations_features))]\n",
    "    df_observations_labels = df_observations_labels[0]\n",
    "    # make each element a list\n",
    "    df_observations_labels = [ [df_observations_labels[i]] for i in range(len(df_observations_labels))]\n",
    "    return df_observations_features, df_observations_labels\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def process_data_user_study(df_observations_features, df_observations_labels, REMOVE_S_AND_R, SEQUENCE_MODE,\n",
    " SPLIT_BY_USER, ADD_PREVIOUS_STATES, PREDICT_ACTION, NORMALIZE_DATA,\n",
    "  test_percentage, val_percentage, previous_states_to_keep):\n",
    "\n",
    "    def process_session_list(df_features_list, df_labels_list, previous_states_to_keep):\n",
    "        df_features_subset = []\n",
    "        df_labels_subset = []\n",
    "        for i in range(len(df_features_list)):\n",
    "            for k in range(len(df_features_list[i])):\n",
    "                features = []\n",
    "                labels = []\n",
    "                for j in range(len(df_features_list[i][k])):\n",
    "                        features.append(df_features_list[i][k][j])\n",
    "                        labels.append(df_labels_list[i][k][j])\n",
    "                if len(features) > 0:\n",
    "                    df_features_subset.append(np.array(features))\n",
    "                    df_labels_subset.append(np.array(labels))\n",
    "\n",
    "        df_features_subset = np.array(df_features_subset)\n",
    "        df_labels_subset = np.array(df_labels_subset)\n",
    "        df_features_subset_append_prev = []\n",
    "        df_labels_subset_append_prev = []\n",
    "\n",
    "        if ADD_PREVIOUS_STATES:\n",
    "            for k in range(len(df_features_subset)):\n",
    "                features = []\n",
    "                labels = []\n",
    "                for j in range(len(df_features_subset[k])):\n",
    "                    feature_construction= df_features_subset[k][j]\n",
    "                    # get previous 5 labels\n",
    "                    previous_labels = np.ones(previous_states_to_keep)\n",
    "                    previous_measurements = np.zeros(8*previous_states_to_keep)\n",
    "                    idx = 0\n",
    "                    for l in range(j-previous_states_to_keep,j):\n",
    "                        if l >= 0:\n",
    "                            previous_labels[idx] = df_labels_subset[k][l]\n",
    "                            previous_measurements[idx*8:(idx+1)*8] = df_features_subset[k][l][:8]\n",
    "                        idx += 1\n",
    "                    feature_construction = np.concatenate((df_features_subset[k][j], previous_labels, previous_measurements))\n",
    "\n",
    "                    features.append(feature_construction)\n",
    "                    labels.append(df_labels_subset[k][j])\n",
    "                if len(features) > 0:\n",
    "                    df_features_subset_append_prev.append(np.array(features))\n",
    "                    df_labels_subset_append_prev.append(np.array(labels))\n",
    "            return df_features_subset_append_prev, df_labels_subset_append_prev\n",
    "\n",
    "        return df_features_subset, df_labels_subset\n",
    "\n",
    "\n",
    "    if not SPLIT_BY_USER:\n",
    "        df_features_subset, df_labels_subset = process_session_list(df_observations_features, df_observations_labels, previous_states_to_keep)\n",
    "        X_train, X_test, y_train, y_test = train_test_split(df_features_subset, df_labels_subset, test_size=test_percentage, random_state=66)\n",
    "        # split into validation and test\n",
    "        X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=val_percentage/(1-test_percentage), random_state=66)\n",
    "        # unflatten X_train and X_test and y_train and y_test\n",
    "        if not SEQUENCE_MODE:\n",
    "            X_train = np.array(list(chain.from_iterable(X_train)))\n",
    "            X_test = np.array(list(chain.from_iterable(X_test)))\n",
    "            y_train = np.array(list(chain.from_iterable(y_train)))\n",
    "            y_test = np.array(list(chain.from_iterable(y_test)))\n",
    "            X_val = np.array(list(chain.from_iterable(X_val)))\n",
    "            y_val = np.array(list(chain.from_iterable(y_val)))\n",
    "            if NORMALIZE_DATA:\n",
    "                scaler = preprocessing.StandardScaler().fit(X_train)\n",
    "                X_train = scaler.transform(X_train)\n",
    "                X_test = scaler.transform(X_test)\n",
    "                X_val = scaler.transform(X_val)\n",
    "    else:\n",
    "        # split on users first\n",
    "        X_train, X_test, y_train, y_test = train_test_split(df_observations_features, df_observations_labels, test_size=test_percentage, random_state=66)\n",
    "        # split into validation and test\n",
    "        X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=val_percentage/(1-test_percentage), random_state=66)\n",
    "        # print lenght of train test and val\n",
    "        X_train, y_train = process_session_list(X_train, y_train, previous_states_to_keep)\n",
    "        X_test, y_test = process_session_list(X_test, y_test,  previous_states_to_keep)\n",
    "        X_val, y_val = process_session_list(X_val, y_val, previous_states_to_keep)\n",
    "\n",
    "        if not SEQUENCE_MODE:\n",
    "            X_train = np.array(list(chain.from_iterable(X_train)))\n",
    "            X_test = np.array(list(chain.from_iterable(X_test)))\n",
    "            y_train = np.array(list(chain.from_iterable(y_train)))\n",
    "            y_test = np.array(list(chain.from_iterable(y_test)))\n",
    "            X_val = np.array(list(chain.from_iterable(X_val)))\n",
    "            y_val = np.array(list(chain.from_iterable(y_val)))\n",
    "            if NORMALIZE_DATA:\n",
    "                scaler = preprocessing.StandardScaler().fit(X_train)\n",
    "                X_train = scaler.transform(X_train)\n",
    "                X_test = scaler.transform(X_test)\n",
    "                X_val = scaler.transform(X_val)\n",
    "    # print lenght of train test and val\n",
    "    print('X_train: ', len(X_train))\n",
    "    print('X_test: ', len(X_test))\n",
    "    print('X_val: ', len(X_val))\n",
    "    return X_train, X_test, X_val, y_train, y_test, y_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '../data/featureframe_user_study.pkl'\n",
    "data = pickle.load(open(path, 'rb'))\n",
    "splitbyusers = True\n",
    "output_path = '.'\n",
    "test_percentage = 0.3\n",
    "val_percentage = 0.1\n",
    "REMOVE_S_AND_R = True # remove shown and replay\n",
    "features_to_keep = np.array([0,3,4,5,8,9, 10])\n",
    "label_index = np.array([11])\n",
    "\n",
    "feature_dict = {'Measurements: compCharLen, confidence, documentLength, numLines, numTokens, promptCharLen, promptEndPos, quantile': 0,\n",
    "'edit percentage': 1, 'time_in_state': 2, 'session_features':3, 'suggestion_label':4, 'prompt_label':5,\n",
    "'suggestion_embedding':6, 'prompt_embedding':7, 'suggestion_text_features':8, 'prompt_text_features':9, 'user_id':10, 'statename':11, 'labeled_state': 12}\n",
    "\n",
    "df_observations_features, df_observations_labels = get_features_labels_user_study(path, features_to_keep, label_index, REMOVE_S_AND_R)\n",
    "\n",
    "\n",
    "# split into train and test\n",
    "SEQUENCE_MODE = False # keep session as a sequence or split it into events\n",
    "SPLIT_BY_USER = bool(splitbyusers) # otherwise split by session uniformly\n",
    "ADD_PREVIOUS_STATES = True\n",
    "PREDICT_ACTION = True # Otherwise predict time in state\n",
    "NORMALIZE_DATA = False # normalize data\n",
    "\n",
    "previous_states_to_keep = 5\n",
    "if not PREDICT_ACTION and SPLIT_BY_USER:\n",
    "    raise ValueError('Cannot predict time and split by user')\n",
    "\n",
    "\n",
    "\n",
    "X_train, X_test, X_val, y_train, y_test, y_val = process_data_user_study(df_observations_features, df_observations_labels,\n",
    "REMOVE_S_AND_R, SEQUENCE_MODE, SPLIT_BY_USER, ADD_PREVIOUS_STATES, PREDICT_ACTION, NORMALIZE_DATA,\n",
    "test_percentage, val_percentage, previous_states_to_keep)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "model = XGBClassifier()\n",
    "model.fit(X_train, y_train)\n",
    "# predict\n",
    "y_pred = model.predict(X_test)\n",
    "# evaluate\n",
    "print(\"Accuracy:\", accuracy_score(y_test, y_pred))\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "confusion_matrix_act = confusion_matrix(y_test, y_pred)\n",
    "classification_report_act = classification_report(y_test, y_pred)\n",
    "print(\"Confusion Matrix:\")\n",
    "print(confusion_matrix(y_test, y_pred))\n",
    "print(\"Classification Report:\")\n",
    "print(classification_report(y_test, y_pred))\n",
    "y_pred_proba = model.predict_proba(X_test)\n",
    "y_pred_proba = y_pred_proba[:,1]\n",
    "print(\"AUC:\", roc_auc_score(y_test, y_pred_proba))\n",
    "auc = roc_auc_score(y_test, y_pred_proba)\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "hussein",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}