{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "245 valid participants\n",
      "{'control': 61, 'dashboard': 61, 'chatxai': 62, 'chatxaiboost': 61, 'chatxaiAuto': 0}\n",
      "91 participants blindly rely on AI advice\n",
      "{'control': 8, 'dashboard': 27, 'chatxai': 32, 'chatxaiboost': 24, 'chatxaiAuto': 0}\n",
      "61 valid participants\n",
      "{'control': 0, 'dashboard': 0, 'chatxai': 0, 'chatxaiboost': 0, 'chatxaiAuto': 61}\n",
      "25 participants blindly rely on AI advice\n",
      "{'control': 0, 'dashboard': 0, 'chatxai': 0, 'chatxaiboost': 0, 'chatxaiAuto': 25}\n",
      "For all users, M: 3.28, SD: 0.68\n",
      "control 61\n",
      "M: 3.15, SD: 0.72\n",
      "dashboard 61\n",
      "M: 3.33, SD: 0.66\n",
      "chatxai 62\n",
      "M: 3.20, SD: 0.63\n",
      "chatxaiboost 61\n",
      "M: 3.28, SD: 0.67\n",
      "chatxaiAuto 61\n",
      "M: 3.44, SD: 0.71\n"
     ]
    }
   ],
   "source": [
    "from util import load_user_data\n",
    "import pandas as pd\n",
    "import sys\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "valid_users, tp_data, _ = load_user_data(filename=\"../data/xailabdata_all.csv\")\n",
    "valid_users_2, tp_data_2, _ = load_user_data(filename=\"../data/xailabdata_llm_agent.csv\")\n",
    "\n",
    "valid_users = valid_users | valid_users_2\n",
    "tp_data.update(tp_data_2)\n",
    "\n",
    "all_conditions = [\"control\", \"dashboard\", \"chatxai\", \"chatxaiboost\", \"chatxaiAuto\"]\n",
    "engagement_list = []\n",
    "engagement_dict = {}\n",
    "variable_dict = {}\n",
    "variable_dict[\"condition\"] = []\n",
    "variable_dict[\"engagement\"] = []\n",
    "for dimension in [\"Propensity to Trust\", \"Familiarity\", \"ATI\", \"mlbackground\"]:\n",
    "    variable_dict[dimension] = []\n",
    "for condition in all_conditions:\n",
    "    engagement_dict[condition] = []\n",
    "for user in valid_users:\n",
    "    tp_condition = tp_data[user][\"condition\"]\n",
    "    trust = tp_data[user][\"Trust_in_automation\"]\n",
    "    # explanation_understanding = tp_data[user][\"explanation_understanding\"]\n",
    "    variable_dict[\"condition\"].append(tp_condition)\n",
    "    user_engagement = tp_data[user][\"user_engagement_scale\"]\n",
    "    variable_dict[\"engagement\"].append(user_engagement)\n",
    "    for dimension in [\"Propensity to Trust\", \"Familiarity\"]:\n",
    "        variable_dict[dimension].append(trust[dimension])\n",
    "    for dimension in [\"ATI\", \"mlbackground\"]:\n",
    "        variable_dict[dimension].append(tp_data[user][dimension])\n",
    "    \n",
    "    engagement_dict[tp_condition].append(user_engagement)\n",
    "    engagement_list.append(user_engagement)\n",
    "print(\"For all users, M: {:.2f}, SD: {:.2f}\".format(np.mean(engagement_list), np.std(engagement_list)))\n",
    "for condition in all_conditions:\n",
    "    print(condition, len(engagement_dict[condition]))\n",
    "    print(\"M: {:.2f}, SD: {:.2f}\".format(np.mean(engagement_dict[condition]), np.std(engagement_dict[condition])))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import kruskal, mannwhitneyu\n",
    "\n",
    "def post_hoc_comparison(data_list_1, data_list_2, name1, name2):\n",
    "\tprint(\"Use pots-hoc analysis\")\n",
    "\tthreshold = 0.05 / 4\n",
    "\tflag = False\n",
    "\tstatistic, pvalue = mannwhitneyu(data_list_1, data_list_2, alternative='greater')\n",
    "\tif pvalue < threshold:\n",
    "\t\tprint(\"Alternative {} > {},\".format(name1, name2), \"pvalue %.4f\"%pvalue, \"statistic %.4f\"%statistic)\n",
    "\t\tflag = True\n",
    "\tstatistic, pvalue = mannwhitneyu(data_list_1, data_list_2, alternative='less')\n",
    "\tif pvalue < threshold:\n",
    "\t\tprint(\"Alternative {} < {},\".format(name1, name2), \"pvalue %.4f\"%pvalue, \"statistic %.4f\"%statistic)\n",
    "\t\tflag = True\n",
    "\tif not flag:\n",
    "\t\tprint(\"No significant difference with post-hoc analysis\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For all participants, compare with experimental conditions\n",
      "5\n",
      "kruskal test result: H:7.14, p:0.129\n",
      "61, Mean: M(control):3.15, SD(control):0.72\n",
      "61, Mean: M(dashboard):3.33, SD(dashboard):0.66\n",
      "62, Mean: M(chatxai):3.20, SD(chatxai):0.63\n",
      "61, Mean: M(chatxaiboost):3.28, SD(chatxaiboost):0.67\n",
      "61, Mean: M(chatxaiAuto):3.44, SD(chatxaiAuto):0.71\n",
      "&7.14 & 0.129& $3.15 \\pm 0.72$ &$3.33 \\pm 0.66$ &$3.20 \\pm 0.63$ &$3.28 \\pm 0.67$ &$3.44 \\pm 0.71$ &\n",
      "-----------------\n"
     ]
    }
   ],
   "source": [
    "print(\"For all participants, compare with experimental conditions\")\n",
    "kwargs = [engagement_dict[condition] for condition in all_conditions]\n",
    "print(len(kwargs))\n",
    "statistic, pvalue = kruskal(*kwargs)\n",
    "print(\"kruskal test result: H:{:.2f}, p:{:.3f}\".format(statistic, pvalue))\n",
    "tp_str = \"&\" + \"{:.2f} & {:.3f}& \".format(statistic, pvalue)\n",
    "for condition in all_conditions:\n",
    "    data_list_1 = engagement_dict[condition]\n",
    "    print(\"{}, Mean: M({}):{:.2f}, SD({}):{:.2f}\".format(len(data_list_1), condition, np.mean(data_list_1), condition, np.std(data_list_1)))\n",
    "    tp_str += \"${:.2f} \\\\pm {:.2f}$ &\".format(np.mean(data_list_1), np.std(data_list_1))\n",
    "print(tp_str)\n",
    "# In the prior test, shall we use 0.05 and then reach a conclusion with calibrated threshold?\n",
    "if pvalue < 0.05 / 4:\n",
    "    length = len(all_conditions)\n",
    "    for i in range(length - 1):\n",
    "        for j in range(i+1, length):\n",
    "            group_1 = all_conditions[i]\n",
    "            group_2 = all_conditions[j]\n",
    "            data_list_1 = engagement_dict[group_1]\n",
    "            data_list_2 = engagement_dict[group_2]\n",
    "            post_hoc_comparison(data_list_1, data_list_2, group_1, group_2)\n",
    "print(\"-\" * 17)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(306, 6)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>condition</th>\n",
       "      <th>engagement</th>\n",
       "      <th>Propensity to Trust</th>\n",
       "      <th>Familiarity</th>\n",
       "      <th>ATI</th>\n",
       "      <th>mlbackground</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>chatxaiAuto</td>\n",
       "      <td>4.416667</td>\n",
       "      <td>3.333333</td>\n",
       "      <td>1.5</td>\n",
       "      <td>1.555556</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>chatxaiAuto</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>3.666667</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.777778</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dashboard</td>\n",
       "      <td>3.500000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.555556</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>chatxaiboost</td>\n",
       "      <td>3.666667</td>\n",
       "      <td>3.333333</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.666667</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>chatxaiAuto</td>\n",
       "      <td>4.750000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.888889</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      condition  engagement  Propensity to Trust  Familiarity       ATI  \\\n",
       "0   chatxaiAuto    4.416667             3.333333          1.5  1.555556   \n",
       "1   chatxaiAuto    4.000000             3.666667          3.0  4.777778   \n",
       "2     dashboard    3.500000             3.000000          2.0  3.555556   \n",
       "3  chatxaiboost    3.666667             3.333333          2.0  3.666667   \n",
       "4   chatxaiAuto    4.750000             5.000000          4.0  4.888889   \n",
       "\n",
       "   mlbackground  \n",
       "0             0  \n",
       "1             1  \n",
       "2             0  \n",
       "3             1  \n",
       "4             1  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(variable_dict)\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------\n",
      "(306, 6)\n",
      "For all participants, compare with experimental conditions\n",
      "engagement\n",
      "      Source  ddof1  ddof2     F  p-unc     n2\n",
      "0  condition      4    301  1.63  0.167  0.021\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ghe/opt/anaconda3/envs/agreement_phi/lib/python3.8/site-packages/outdated/utils.py:14: OutdatedPackageWarning: The package pingouin is out of date. Your version is 0.5.3, the latest is 0.5.5.\n",
      "Set the environment variable OUTDATED_IGNORE=1 to disable these warnings.\n",
      "  return warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                Source       SS   DF       F  p-unc     n2\n",
      "0            condition    2.656    4   1.957  0.101  0.020\n",
      "1  Propensity to Trust   21.740    1  64.085  0.000  0.166\n",
      "2          Familiarity    1.136    1   3.349  0.068  0.009\n",
      "3                  ATI    4.469    1  13.173  0.000  0.034\n",
      "4         mlbackground    0.484    1   1.427  0.233  0.004\n",
      "5             Residual  100.755  297     NaN    NaN    NaN\n",
      "1.96 & .101 & 64.09 & .000 & 3.35 & .068 & 13.17 & .000 & 1.43 & .233\\\\\n",
      "condition\n",
      "chatxai         3.20\n",
      "chatxaiAuto     3.44\n",
      "chatxaiboost    3.28\n",
      "control         3.15\n",
      "dashboard       3.33\n",
      "Name: engagement, dtype: float64\n",
      "condition\n",
      "chatxai         0.63\n",
      "chatxaiAuto     0.72\n",
      "chatxaiboost    0.67\n",
      "control         0.72\n",
      "dashboard       0.66\n",
      "Name: engagement, dtype: float64\n",
      "-----------------\n"
     ]
    }
   ],
   "source": [
    "print(\"-\" * 34)\n",
    "df = pd.DataFrame(variable_dict)\n",
    "print(df.shape)\n",
    "from pingouin import ancova, anova\n",
    "print(\"For all participants, compare with experimental conditions\")\n",
    "# for dimension in dimensions:\n",
    "dimension = \"engagement\"\n",
    "print(dimension)\n",
    "aov = anova(dv=dimension, between='condition', data=df, effsize='n2')\n",
    "print(aov.round(3))\n",
    "aov = ancova(dv=\"engagement\", covar=[\"Propensity to Trust\", \"Familiarity\", \"ATI\", \"mlbackground\"], between='condition', data=df, effsize='n2')\n",
    "print(aov.round(3))\n",
    "tp_dict = aov.to_dict()\n",
    "# f_list = [\"{:.2f}\".format(tp_dict['F'][index]) for index in range(0, 5)]\n",
    "# p_list = [\"{:.2f}\".format(tp_dict['p-unc'][index]) for index in range(0, 5)]\n",
    "tp_str = \"{:.2f} & {}\".format(tp_dict['F'][0], \"{:.3f}\".format(tp_dict['p-unc'][0])[1:])\n",
    "for index in range(1,5):\n",
    "    tp_str += \" & {:.2f} & {}\".format(tp_dict['F'][index], \"{:.3f}\".format(tp_dict['p-unc'][index])[1:])\n",
    "tp_str += \"\\\\\\\\\"\n",
    "print(tp_str)\n",
    "print(df.groupby(['condition'])[dimension].mean().round(2))\n",
    "print(df.groupby(['condition'])[dimension].std().round(2))\n",
    "print(\"-\" * 17)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "agreement_phi",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
