{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from util import load_user_data\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "245 valid participants\n",
      "{'control': 61, 'dashboard': 61, 'chatxai': 62, 'chatxaiboost': 61, 'chatxaiAuto': 0}\n",
      "91 participants blindly rely on AI advice\n",
      "{'control': 8, 'dashboard': 27, 'chatxai': 32, 'chatxaiboost': 24, 'chatxaiAuto': 0}\n",
      "61 valid participants\n",
      "{'control': 0, 'dashboard': 0, 'chatxai': 0, 'chatxaiboost': 0, 'chatxaiAuto': 61}\n",
      "25 participants blindly rely on AI advice\n",
      "{'control': 0, 'dashboard': 0, 'chatxai': 0, 'chatxaiboost': 0, 'chatxaiAuto': 25}\n",
      "{'control': {'Reliability/Competence': [], 'Understandability/Predictability': [], 'Trust in Automation': []}, 'dashboard': {'Reliability/Competence': [], 'Understandability/Predictability': [], 'Trust in Automation': []}, 'chatxai': {'Reliability/Competence': [], 'Understandability/Predictability': [], 'Trust in Automation': []}, 'chatxaiboost': {'Reliability/Competence': [], 'Understandability/Predictability': [], 'Trust in Automation': []}, 'chatxaiAuto': {'Reliability/Competence': [], 'Understandability/Predictability': [], 'Trust in Automation': []}}\n",
      "(306, 9)\n"
     ]
    }
   ],
   "source": [
    "valid_users, tp_data, _ = load_user_data(filename=\"../data/xailabdata_all.csv\")\n",
    "valid_users_2, tp_data_2, _ = load_user_data(filename=\"../data/xailabdata_llm_agent.csv\")\n",
    "\n",
    "valid_users = valid_users | valid_users_2\n",
    "tp_data.update(tp_data_2)\n",
    "variable_dict = {}\n",
    "variable_dict[\"condition\"] = []\n",
    "variable_dict[\"engagement\"] = []\n",
    "dimensions = [\"Reliability/Competence\", \"Understandability/Predictability\", \"Trust in Automation\"]\n",
    "covariates = [\"ATI\", \"mlbackground\"]\n",
    "all_conditions = [\"control\", \"dashboard\", \"chatxai\", \"chatxaiboost\", \"chatxaiAuto\"]\n",
    "condition_dict = {}\n",
    "for dimension in dimensions:\n",
    "    variable_dict[dimension] = []\n",
    "for condition in all_conditions:\n",
    "    condition_dict[condition] = {}\n",
    "    for dimension in dimensions:\n",
    "        condition_dict[condition][dimension] = []\n",
    "for dimension in [\"Propensity to Trust\", \"Familiarity\", \"ATI\", \"mlbackground\"]:\n",
    "    variable_dict[dimension] = []\n",
    "print(condition_dict)\n",
    "for user in valid_users:\n",
    "    tp_condition = tp_data[user][\"condition\"]\n",
    "    # if tp_condition == \"control\":\n",
    "    # \tcontinue\n",
    "    trust = tp_data[user][\"Trust_in_automation\"]\n",
    "    variable_dict[\"condition\"].append(tp_condition)\n",
    "    user_engagement = tp_data[user][\"user_engagement_scale\"]\n",
    "    variable_dict[\"engagement\"].append(user_engagement)\n",
    "    for dimension in dimensions:\n",
    "        variable_dict[dimension].append(trust[dimension])\n",
    "        condition_dict[tp_condition][dimension].append(trust[dimension])\n",
    "    for dimension in [\"Propensity to Trust\", \"Familiarity\"]:\n",
    "        variable_dict[dimension].append(trust[dimension])\n",
    "    for dimension in [\"ATI\", \"mlbackground\"]:\n",
    "        variable_dict[dimension].append(tp_data[user][dimension])\n",
    "\n",
    "df = pd.DataFrame(variable_dict)\n",
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ghe/opt/anaconda3/envs/agreement_phi/lib/python3.8/site-packages/outdated/utils.py:14: OutdatedPackageWarning: The package pingouin is out of date. Your version is 0.5.3, the latest is 0.5.5.\n",
      "Set the environment variable OUTDATED_IGNORE=1 to disable these warnings.\n",
      "  return warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For all participants, compare with experimental conditions\n",
      "Reliability/Competence\n",
      "                Source      SS   DF        F  p-unc     n2\n",
      "0            condition   3.066    4    2.901  0.022  0.018\n",
      "1  Propensity to Trust  84.303    1  319.098  0.000  0.505\n",
      "2          Familiarity   0.244    1    0.923  0.337  0.001\n",
      "3                  ATI   0.632    1    2.392  0.123  0.004\n",
      "4         mlbackground   0.252    1    0.953  0.330  0.002\n",
      "5             Residual  78.465  297      NaN    NaN    NaN\n",
      "      Multiple Comparison of Means - Tukey HSD, FWER=0.01      \n",
      "===============================================================\n",
      "   group1       group2    meandiff p-adj   lower  upper  reject\n",
      "---------------------------------------------------------------\n",
      "     chatxai  chatxaiAuto  -0.2467 0.3683 -0.6842 0.1909  False\n",
      "     chatxai chatxaiboost   0.1277 0.8818 -0.3099 0.5652  False\n",
      "     chatxai      control  -0.3177  0.137 -0.7552 0.1199  False\n",
      "     chatxai    dashboard   0.0156    1.0 -0.4219 0.4532  False\n",
      " chatxaiAuto chatxaiboost   0.3743 0.0507  -0.065 0.8136  False\n",
      " chatxaiAuto      control   -0.071 0.9853 -0.5104 0.3683  False\n",
      " chatxaiAuto    dashboard   0.2623 0.3092  -0.177 0.7016  False\n",
      "chatxaiboost      control  -0.4454 0.0108 -0.8847 -0.006   True\n",
      "chatxaiboost    dashboard   -0.112 0.9244 -0.5513 0.3273  False\n",
      "     control    dashboard   0.3333 0.1078  -0.106 0.7726  False\n",
      "---------------------------------------------------------------\n",
      "control 2.969945355191258\n",
      "dashboard 3.3032786885245895\n",
      "chatxai 3.2876344086021505\n",
      "chatxaiboost 3.415300546448087\n",
      "chatxaiAuto 3.040983606557377\n",
      "-----------------\n",
      "Understandability/Predictability\n",
      "                Source       SS   DF        F  p-unc     n2\n",
      "0            condition   10.122    4    5.773  0.000  0.052\n",
      "1  Propensity to Trust   47.831    1  109.123  0.000  0.247\n",
      "2          Familiarity    0.885    1    2.018  0.156  0.005\n",
      "3                  ATI    4.416    1   10.074  0.002  0.023\n",
      "4         mlbackground    0.212    1    0.483  0.488  0.001\n",
      "5             Residual  130.181  297      NaN    NaN    NaN\n",
      "      Multiple Comparison of Means - Tukey HSD, FWER=0.01       \n",
      "================================================================\n",
      "   group1       group2    meandiff p-adj   lower   upper  reject\n",
      "----------------------------------------------------------------\n",
      "     chatxai  chatxaiAuto  -0.4175 0.0302 -0.8764  0.0414  False\n",
      "     chatxai chatxaiboost   0.0128    1.0  -0.446  0.4717  False\n",
      "     chatxai      control  -0.6019 0.0003 -1.0608 -0.1431   True\n",
      "     chatxai    dashboard  -0.1101 0.9386  -0.569  0.3487  False\n",
      " chatxaiAuto chatxaiboost   0.4303  0.024 -0.0304  0.8911  False\n",
      " chatxaiAuto      control  -0.1844 0.6995 -0.6452  0.2763  False\n",
      " chatxaiAuto    dashboard   0.3074  0.204 -0.1533  0.7681  False\n",
      "chatxaiboost      control  -0.6148 0.0002 -1.0755  -0.154   True\n",
      "chatxaiboost    dashboard   -0.123 0.9119 -0.5837  0.3378  False\n",
      "     control    dashboard   0.4918 0.0061  0.0311  0.9525   True\n",
      "----------------------------------------------------------------\n",
      "control 3.180327868852459\n",
      "dashboard 3.6721311475409837\n",
      "chatxai 3.782258064516129\n",
      "chatxaiboost 3.7950819672131146\n",
      "chatxaiAuto 3.3647540983606556\n",
      "-----------------\n",
      "Trust in Automation\n",
      "                Source       SS   DF        F  p-unc     n2\n",
      "0            condition    3.862    4    2.841  0.025  0.015\n",
      "1  Propensity to Trust  148.615    1  437.349  0.000  0.585\n",
      "2          Familiarity    0.371    1    1.092  0.297  0.001\n",
      "3                  ATI    0.171    1    0.503  0.479  0.001\n",
      "4         mlbackground    0.102    1    0.301  0.584  0.000\n",
      "5             Residual  100.923  297      NaN    NaN    NaN\n",
      "      Multiple Comparison of Means - Tukey HSD, FWER=0.01      \n",
      "===============================================================\n",
      "   group1       group2    meandiff p-adj   lower  upper  reject\n",
      "---------------------------------------------------------------\n",
      "     chatxai  chatxaiAuto   -0.149 0.9065 -0.6975 0.3995  False\n",
      "     chatxai chatxaiboost   -0.026 0.9999 -0.5745 0.5224  False\n",
      "     chatxai      control  -0.5342 0.0163 -1.0827 0.0142  False\n",
      "     chatxai    dashboard  -0.0015    1.0 -0.5499  0.547  False\n",
      " chatxaiAuto chatxaiboost    0.123 0.9523 -0.4277 0.6736  False\n",
      " chatxaiAuto      control  -0.3852 0.1647 -0.9359 0.1654  False\n",
      " chatxaiAuto    dashboard   0.1475 0.9107 -0.4031 0.6982  False\n",
      "chatxaiboost      control  -0.5082 0.0267 -1.0589 0.0425  False\n",
      "chatxaiboost    dashboard   0.0246 0.9999 -0.5261 0.5753  False\n",
      "     control    dashboard   0.5328 0.0174 -0.0179 1.0835  False\n",
      "---------------------------------------------------------------\n",
      "control 2.877049180327869\n",
      "dashboard 3.4098360655737703\n",
      "chatxai 3.411290322580645\n",
      "chatxaiboost 3.3852459016393444\n",
      "chatxaiAuto 3.262295081967213\n",
      "-----------------\n"
     ]
    }
   ],
   "source": [
    "print(\"-\" * 34)\n",
    "from pingouin import ancova, anova\n",
    "from statsmodels.stats.multicomp import pairwise_tukeyhsd\n",
    "print(\"For all participants, compare with experimental conditions\")\n",
    "for dimension in dimensions:\n",
    "    print(dimension)\n",
    "    # aov = anova(dv=dimension, between='condition', data=df, effsize='n2')\n",
    "    aov = ancova(dv=dimension, covar=[\"Propensity to Trust\", \"Familiarity\", \"ATI\", \"mlbackground\"], between='condition', data=df, effsize='n2')\n",
    "    print(aov.round(3))\n",
    "    if aov.to_dict()['p-unc'][0] <= 0.05:\n",
    "        tukey = pairwise_tukeyhsd(endog=df[dimension], groups=df['condition'], alpha=0.0125)\n",
    "        print(tukey)\n",
    "    for condition in all_conditions:\n",
    "        print(condition, np.mean(condition_dict[condition][dimension]))\n",
    "    print(\"-\" * 17)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "User Engagement\n",
      "                Source       SS   DF       F  p-unc     n2\n",
      "0            condition    2.656    4   1.957  0.101  0.020\n",
      "1  Propensity to Trust   21.740    1  64.085  0.000  0.166\n",
      "2          Familiarity    1.136    1   3.349  0.068  0.009\n",
      "3                  ATI    4.469    1  13.173  0.000  0.034\n",
      "4         mlbackground    0.484    1   1.427  0.233  0.004\n",
      "5             Residual  100.755  297     NaN    NaN    NaN\n"
     ]
    }
   ],
   "source": [
    "print(\"User Engagement\")\n",
    "aov = ancova(dv=\"engagement\", covar=[\"Propensity to Trust\", \"Familiarity\", \"ATI\", \"mlbackground\"], between='condition', data=df, effsize='n2')\n",
    "print(aov.round(3))\n",
    "if aov.to_dict()['p-unc'][0] <= 0.05:\n",
    "    tukey = pairwise_tukeyhsd(endog=df[\"engagement\"], groups=df['condition'], alpha=0.0125)\n",
    "    print(tukey)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "agreement_phi",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
