{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "245 valid participants\n",
      "{'control': 61, 'dashboard': 61, 'chatxai': 62, 'chatxaiboost': 61, 'chatxaiAuto': 0}\n",
      "91 participants blindly rely on AI advice\n",
      "{'control': 8, 'dashboard': 27, 'chatxai': 32, 'chatxaiboost': 24, 'chatxaiAuto': 0}\n",
      "61 valid participants\n",
      "{'control': 0, 'dashboard': 0, 'chatxai': 0, 'chatxaiboost': 0, 'chatxaiAuto': 61}\n",
      "25 participants blindly rely on AI advice\n",
      "{'control': 0, 'dashboard': 0, 'chatxai': 0, 'chatxaiboost': 0, 'chatxaiAuto': 25}\n",
      "(306, 14)\n"
     ]
    }
   ],
   "source": [
    "from util import load_user_data\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# Load explanation understanding, no control condition\n",
    "valid_users, tp_data, _ = load_user_data(filename=\"../data/xailabdata_all.csv\")\n",
    "valid_users_2, tp_data_2, _ = load_user_data(filename=\"../data/xailabdata_llm_agent.csv\")\n",
    "\n",
    "valid_users = valid_users | valid_users_2\n",
    "tp_data.update(tp_data_2)\n",
    "\n",
    "variable_dict = {}\n",
    "variable_dict[\"condition\"] = []\n",
    "understanding_variable_dict = {}\n",
    "understanding_variable_dict[\"condition\"] = []\n",
    "explanation_understanding_dimensions = [\"perceived_feature_understanding\", \"completeness\", \"coherence\", \"clarity\", \\\n",
    "    \"learning_effect\", \"understanding_of_system\", \"usefulness_explanation\", \"objective_feature_understanding\"]\n",
    "covariates_dimensions = [\"Propensity to Trust\", \"Familiarity\", \"ATI\", \"mlbackground\"]\n",
    "trust_dimensions = [\"Reliability/Competence\", \"Understandability/Predictability\", \"Trust in Automation\"]\n",
    "reliance_dimensions = [\"accuracy\", \"agreement_fraction\", \"switch_fraction\", \"accuracy-wid\", \"RAIR\", \"RSR\"]\n",
    "for dimension in reliance_dimensions:\n",
    "    variable_dict[dimension] = []\n",
    "for dimension in trust_dimensions:\n",
    "    variable_dict[dimension] = []\n",
    "for dimension in covariates_dimensions:\n",
    "    variable_dict[dimension] = []\n",
    "    understanding_variable_dict[dimension] = []\n",
    "for dimension in explanation_understanding_dimensions:\n",
    "    understanding_variable_dict[dimension] = []\n",
    "for user in valid_users:\n",
    "    tp_condition = tp_data[user][\"condition\"]\n",
    "    variable_dict[\"condition\"].append(tp_condition)\n",
    "\n",
    "    # for users in all conditions, load reliance\n",
    "    reliance_measures = tp_data[user][\"reliance_measures\"]\n",
    "    for dimension in reliance_dimensions:\n",
    "        variable_dict[dimension].append(reliance_measures[dimension])\n",
    "    # for participants in all conditions, load covariates and trust\n",
    "    trust = tp_data[user][\"Trust_in_automation\"]\n",
    "    for dimension in trust_dimensions:\n",
    "        variable_dict[dimension].append(trust[dimension])\n",
    "    for dimension in [\"Propensity to Trust\", \"Familiarity\"]:\n",
    "        variable_dict[dimension].append(trust[dimension])\n",
    "    for dimension in [\"ATI\", \"mlbackground\"]:\n",
    "        variable_dict[dimension].append(tp_data[user][dimension])\n",
    "    if tp_condition == \"control\":\n",
    "        continue\n",
    "    # for participants in dashboard, chatxai and chatxaiboost conditions, load explanation udnerstanding\n",
    "    explanation_understanding = tp_data[user][\"explanation_understanding\"]\n",
    "    understanding_variable_dict[\"condition\"].append(tp_condition)\n",
    "    for dimension in explanation_understanding_dimensions:\n",
    "        understanding_variable_dict[dimension].append(explanation_understanding[dimension])\n",
    "    for dimension in [\"Propensity to Trust\", \"Familiarity\"]:\n",
    "        understanding_variable_dict[dimension].append(trust[dimension])\n",
    "    for dimension in [\"ATI\", \"mlbackground\"]:\n",
    "        understanding_variable_dict[dimension].append(tp_data[user][dimension])\n",
    "\n",
    "# for dimension in variable_dict:\n",
    "#     print(dimension, len(variable_dict[dimension]))\n",
    "\n",
    "df = pd.DataFrame(variable_dict)\n",
    "df2 = pd.DataFrame(understanding_variable_dict)\n",
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>condition</th>\n",
       "      <th>Propensity to Trust</th>\n",
       "      <th>Familiarity</th>\n",
       "      <th>ATI</th>\n",
       "      <th>mlbackground</th>\n",
       "      <th>perceived_feature_understanding</th>\n",
       "      <th>completeness</th>\n",
       "      <th>coherence</th>\n",
       "      <th>clarity</th>\n",
       "      <th>learning_effect</th>\n",
       "      <th>understanding_of_system</th>\n",
       "      <th>usefulness_explanation</th>\n",
       "      <th>objective_feature_understanding</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>chatxai</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.666667</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.941976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>chatxaiboost</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.555556</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>3.5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>3.9</td>\n",
       "      <td>0.993013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>chatxai</td>\n",
       "      <td>3.333333</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>3.9</td>\n",
       "      <td>0.758516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>chatxaiboost</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>3.6</td>\n",
       "      <td>0.716540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>chatxaiAuto</td>\n",
       "      <td>1.333333</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.111111</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2.5</td>\n",
       "      <td>0.802893</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      condition  Propensity to Trust  Familiarity       ATI  mlbackground  \\\n",
       "0       chatxai             3.000000          1.0  3.666667             0   \n",
       "1  chatxaiboost             3.000000          2.0  4.555556             1   \n",
       "2       chatxai             3.333333          2.0  4.000000             0   \n",
       "3  chatxaiboost             3.000000          2.0  3.000000             0   \n",
       "4   chatxaiAuto             1.333333          2.0  5.111111             0   \n",
       "\n",
       "   perceived_feature_understanding  completeness  coherence  clarity  \\\n",
       "0                                5           4.0          4        5   \n",
       "1                                4           3.5          5        5   \n",
       "2                                3           4.0          4        4   \n",
       "3                                4           3.5          3        4   \n",
       "4                                3           3.0          2        3   \n",
       "\n",
       "   learning_effect  understanding_of_system  usefulness_explanation  \\\n",
       "0                4                        4                     5.0   \n",
       "1                5                        4                     3.9   \n",
       "2                4                        3                     3.9   \n",
       "3                4                        4                     3.6   \n",
       "4                2                        2                     2.5   \n",
       "\n",
       "   objective_feature_understanding  \n",
       "0                         0.941976  \n",
       "1                         0.993013  \n",
       "2                         0.758516  \n",
       "3                         0.716540  \n",
       "4                         0.802893  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "perceived_feature_understanding & 0.344 &\\textbf{.000}$^{\\dagger\\dagger}$& 0.131 & .041$^{\\dagger}$& 0.148 & .021$^{\\dagger}$& 0.049 & .444\\\\\n",
      "completeness & 0.366 &\\textbf{.000}$^{\\dagger\\dagger}$& 0.106 & .097& 0.073 & .254& 0.152 & .017$^{\\dagger}$\\\\\n",
      "coherence & 0.387 &\\textbf{.000}$^{\\dagger\\dagger}$& 0.131 & .040$^{\\dagger}$& 0.087 & .175& 0.135 & .035$^{\\dagger}$\\\\\n",
      "clarity & 0.427 &\\textbf{.000}$^{\\dagger\\dagger}$& 0.069 & .285& 0.129 & .044$^{\\dagger}$& 0.142 & .026$^{\\dagger}$\\\\\n",
      "learning_effect & 0.232 &\\textbf{.000}$^{\\dagger\\dagger}$& 0.173 &\\textbf{.007}$^{\\dagger\\dagger}$& 0.115 & .072& 0.147 & .021$^{\\dagger}$\\\\\n",
      "understanding_of_system & 0.343 &\\textbf{.000}$^{\\dagger\\dagger}$& 0.082 & .202& 0.146 & .022$^{\\dagger}$& 0.080 & .210\\\\\n",
      "usefulness_explanation & 0.423 &\\textbf{.000}$^{\\dagger\\dagger}$& 0.166 &\\textbf{.009}$^{\\dagger\\dagger}$& 0.172 &\\textbf{.007}$^{\\dagger\\dagger}$& 0.083 & .196\\\\\n",
      "objective_feature_understanding & 0.108 & .092& -0.152 & .017$^{\\dagger}$& 0.013 & .844& -0.024 & .714\\\\\n",
      "-----------------\n",
      "Reliability/Competence & 0.677 &\\textbf{.000}$^{\\dagger\\dagger}$& 0.126 & .028$^{\\dagger}$& 0.171 &\\textbf{.003}$^{\\dagger\\dagger}$& 0.153 &\\textbf{.008}$^{\\dagger\\dagger}$\\\\\n",
      "Understandability/Predictability & 0.472 &\\textbf{.000}$^{\\dagger\\dagger}$& 0.083 & .150& 0.243 &\\textbf{.000}$^{\\dagger\\dagger}$& 0.158 &\\textbf{.006}$^{\\dagger\\dagger}$\\\\\n",
      "Trust in Automation & 0.774 &\\textbf{.000}$^{\\dagger\\dagger}$& 0.235 &\\textbf{.000}$^{\\dagger\\dagger}$& 0.154 &\\textbf{.007}$^{\\dagger\\dagger}$& 0.164 &\\textbf{.004}$^{\\dagger\\dagger}$\\\\\n",
      "-----------------\n",
      "accuracy & 0.091 & .111& 0.073 & .202& -0.039 & .502& -0.019 & .740\\\\\n",
      "agreement_fraction & 0.223 &\\textbf{.000}$^{\\dagger\\dagger}$& 0.055 & .335& 0.030 & .598& -0.039 & .499\\\\\n",
      "switch_fraction & 0.137 & .016$^{\\dagger}$& -0.030 & .595& -0.001 & .982& 0.037 & .518\\\\\n",
      "accuracy-wid & 0.056 & .326& 0.032 & .582& -0.045 & .434& 0.057 & .322\\\\\n",
      "RAIR & 0.118 & .040$^{\\dagger}$& -0.001 & .980& -0.026 & .648& 0.026 & .654\\\\\n",
      "RSR & -0.186 &\\textbf{.001}$^{\\dagger\\dagger}$& -0.024 & .674& -0.080 & .162& -0.038 & .505\\\\\n",
      "-----------------\n"
     ]
    }
   ],
   "source": [
    "from scipy.stats import spearmanr\n",
    "\n",
    "# table rows: [\"Propensity to Trust\", \"Familiarity\", \"ATI\", \"mlbackground\"]\n",
    "# corr, p; corr, p; corr, p; corr, p\n",
    "# explanation understanding\n",
    "# trust\n",
    "# reliance\n",
    "\n",
    "str_dict = {}\n",
    "for dv in explanation_understanding_dimensions:\n",
    "    str_dict[dv] = \"{} \".format(dv)\n",
    "    for cov in covariates_dimensions:\n",
    "        correlation, pvalue = spearmanr(understanding_variable_dict[cov], understanding_variable_dict[dv])\n",
    "        # if pvalue < 0.05 / 4:\n",
    "        # print(\"Variable {} and variable {} have spearman correlation {:.3f} and pvalue {:.3f}\".format(cov, dv, correlation, pvalue))\n",
    "        # str_dict[dv] += \"& {} & {}\".format(\"%.3f\"%correlation, (\"%.3f\"%pvalue)[1:])\n",
    "        if pvalue < 0.0125:\n",
    "            str_dict[dv] += \"& {} &\".format(\"%.3f\"%correlation)\n",
    "            str_dict[dv] += \"\\\\textbf{\"\n",
    "            str_dict[dv] += \"{}\".format((\"%.3f\"%pvalue)[1:])\n",
    "            str_dict[dv] += \"}$^{\\dagger\\dagger}$\"\n",
    "        elif pvalue < 0.05:\n",
    "            str_dict[dv] += \"& {} & {}\".format(\"%.3f\"%correlation, (\"%.3f\"%pvalue)[1:])\n",
    "            str_dict[dv] += \"$^{\\dagger}$\"\n",
    "        else:\n",
    "            str_dict[dv] += \"& {} & {}\".format(\"%.3f\"%correlation, (\"%.3f\"%pvalue)[1:])\n",
    "    str_dict[dv] += \"\\\\\\\\\"\n",
    "    print(str_dict[dv])\n",
    "print(\"-\" * 17)\n",
    "\n",
    "for dv in trust_dimensions:\n",
    "    str_dict[dv] = \"{} \".format(dv)\n",
    "    for cov in covariates_dimensions:\n",
    "        correlation, pvalue = spearmanr(variable_dict[cov], variable_dict[dv])\n",
    "        # if pvalue < 0.05 / 4:\n",
    "        # print(\"Variable {} and variable {} have spearman correlation {:.3f} and pvalue {:.3f}\".format(cov, dv, correlation, pvalue))\n",
    "        # str_dict[dv] += \"& {} & {}\".format(\"%.3f\"%correlation, (\"%.3f\"%pvalue)[1:])\n",
    "        if pvalue < 0.0125:\n",
    "            str_dict[dv] += \"& {} &\".format(\"%.3f\"%correlation)\n",
    "            str_dict[dv] += \"\\\\textbf{\"\n",
    "            str_dict[dv] += \"{}\".format((\"%.3f\"%pvalue)[1:])\n",
    "            str_dict[dv] += \"}$^{\\dagger\\dagger}$\"\n",
    "        elif pvalue < 0.05:\n",
    "            str_dict[dv] += \"& {} & {}\".format(\"%.3f\"%correlation, (\"%.3f\"%pvalue)[1:])\n",
    "            str_dict[dv] += \"$^{\\dagger}$\"\n",
    "        else:\n",
    "            str_dict[dv] += \"& {} & {}\".format(\"%.3f\"%correlation, (\"%.3f\"%pvalue)[1:])\n",
    "    str_dict[dv] += \"\\\\\\\\\"\n",
    "    print(str_dict[dv])\n",
    "print(\"-\" * 17)\n",
    "\n",
    "\n",
    "for dv in reliance_dimensions:\n",
    "    str_dict[dv] = \"{} \".format(dv)\n",
    "    for cov in covariates_dimensions:\n",
    "        correlation, pvalue = spearmanr(variable_dict[cov], variable_dict[dv])\n",
    "        # if pvalue < 0.05 / 4:\n",
    "        # print(\"Variable {} and variable {} have spearman correlation {:.3f} and pvalue {:.3f}\".format(cov, dv, correlation, pvalue))\n",
    "        # str_dict[dv] += \"& {} & {}\".format(\"%.3f\"%correlation, (\"%.3f\"%pvalue)[1:])\n",
    "        if pvalue < 0.0125:\n",
    "            str_dict[dv] += \"& {} &\".format(\"%.3f\"%correlation)\n",
    "            str_dict[dv] += \"\\\\textbf{\"\n",
    "            str_dict[dv] += \"{}\".format((\"%.3f\"%pvalue)[1:])\n",
    "            str_dict[dv] += \"}$^{\\dagger\\dagger}$\"\n",
    "        elif pvalue < 0.05:\n",
    "            str_dict[dv] += \"& {} & {}\".format(\"%.3f\"%correlation, (\"%.3f\"%pvalue)[1:])\n",
    "            str_dict[dv] += \"$^{\\dagger}$\"\n",
    "        else:\n",
    "            str_dict[dv] += \"& {} & {}\".format(\"%.3f\"%correlation, (\"%.3f\"%pvalue)[1:])\n",
    "    str_dict[dv] += \"\\\\\\\\\"\n",
    "    print(str_dict[dv])\n",
    "print(\"-\" * 17)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "agreement_phi",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
