{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "245 valid participants\n",
      "{'control': 61, 'dashboard': 61, 'chatxai': 62, 'chatxaiboost': 61, 'chatxaiAuto': 0}\n",
      "91 participants blindly rely on AI advice\n",
      "{'control': 8, 'dashboard': 27, 'chatxai': 32, 'chatxaiboost': 24, 'chatxaiAuto': 0}\n",
      "61 valid participants\n",
      "{'control': 0, 'dashboard': 0, 'chatxai': 0, 'chatxaiboost': 0, 'chatxaiAuto': 61}\n",
      "25 participants blindly rely on AI advice\n",
      "{'control': 0, 'dashboard': 0, 'chatxai': 0, 'chatxaiboost': 0, 'chatxaiAuto': 25}\n",
      "(245, 14)\n"
     ]
    }
   ],
   "source": [
    "from util import load_user_data\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# Load explanation understanding, no control condition\n",
    "valid_users, tp_data, _ = load_user_data(filename=\"../data/xailabdata_all.csv\")\n",
    "valid_users_2, tp_data_2, _ = load_user_data(filename=\"../data/xailabdata_llm_agent.csv\")\n",
    "\n",
    "valid_users = valid_users | valid_users_2\n",
    "tp_data.update(tp_data_2)\n",
    "variable_dict = {}\n",
    "variable_dict[\"condition\"] = []\n",
    "understanding_variable_dict = {}\n",
    "understanding_variable_dict[\"condition\"] = []\n",
    "explanation_understanding_dimensions = [\"perceived_feature_understanding\", \"completeness\", \"coherence\", \"clarity\", \\\n",
    "    \"learning_effect\", \"understanding_of_system\", \"usefulness_explanation\", \"objective_feature_understanding\"]\n",
    "covariates_dimensions = [\"Propensity to Trust\", \"Familiarity\", \"ATI\", \"mlbackground\"]\n",
    "trust_dimensions = [\"Reliability/Competence\", \"Understandability/Predictability\", \"Trust in Automation\"]\n",
    "reliance_dimensions = [\"accuracy\", \"accuracy-wid\", \"agreement_fraction\", \"switch_fraction\", \"RAIR\", \"RSR\"]\n",
    "for dimension in reliance_dimensions:\n",
    "    variable_dict[dimension] = []\n",
    "for dimension in trust_dimensions:\n",
    "    variable_dict[dimension] = []\n",
    "for dimension in covariates_dimensions:\n",
    "    variable_dict[dimension] = []\n",
    "    understanding_variable_dict[dimension] = []\n",
    "for dimension in explanation_understanding_dimensions:\n",
    "    understanding_variable_dict[dimension] = []\n",
    "for user in valid_users:\n",
    "    tp_condition = tp_data[user][\"condition\"]\n",
    "    if tp_condition == \"control\":\n",
    "        continue\n",
    "    variable_dict[\"condition\"].append(tp_condition)\n",
    "\n",
    "    # for users in all conditions, load reliance\n",
    "    reliance_measures = tp_data[user][\"reliance_measures\"]\n",
    "    for dimension in reliance_dimensions:\n",
    "        variable_dict[dimension].append(reliance_measures[dimension])\n",
    "    # for participants in all conditions, load covariates and trust\n",
    "    trust = tp_data[user][\"Trust_in_automation\"]\n",
    "    for dimension in trust_dimensions:\n",
    "        variable_dict[dimension].append(trust[dimension])\n",
    "    for dimension in [\"Propensity to Trust\", \"Familiarity\"]:\n",
    "        variable_dict[dimension].append(trust[dimension])\n",
    "    for dimension in [\"ATI\", \"mlbackground\"]:\n",
    "        variable_dict[dimension].append(tp_data[user][dimension])\n",
    "    if tp_condition == \"control\":\n",
    "        continue\n",
    "    # for participants in dashboard, chatxai and chatxaiboost conditions, load explanation udnerstanding\n",
    "    explanation_understanding = tp_data[user][\"explanation_understanding\"]\n",
    "    understanding_variable_dict[\"condition\"].append(tp_condition)\n",
    "    for dimension in explanation_understanding_dimensions:\n",
    "        understanding_variable_dict[dimension].append(explanation_understanding[dimension])\n",
    "    for dimension in [\"Propensity to Trust\", \"Familiarity\"]:\n",
    "        understanding_variable_dict[dimension].append(trust[dimension])\n",
    "    for dimension in [\"ATI\", \"mlbackground\"]:\n",
    "        understanding_variable_dict[dimension].append(tp_data[user][dimension])\n",
    "\n",
    "# for dimension in variable_dict:\n",
    "#     print(dimension, len(variable_dict[dimension]))\n",
    "\n",
    "df = pd.DataFrame(variable_dict)\n",
    "df2 = pd.DataFrame(understanding_variable_dict)\n",
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Order for reliance measures ['accuracy', 'accuracy-wid', 'agreement_fraction', 'switch_fraction', 'RAIR', 'RSR']\n",
      "perceived_feature_understanding\n",
      "perceived_feature_understanding & 0.045 & .484& -0.024 & .709& 0.254 &\\textbf{.000}$^{\\dagger\\dagger}$& 0.117 & .067& 0.096 & .135& -0.293 &\\textbf{.000}$^{\\dagger\\dagger}$\\\\\n",
      "completeness\n",
      "completeness & 0.050 & .435& 0.056 & .387& 0.146 & .022{\\dagger}$& 0.142 & .026{\\dagger}$& 0.157 & .014{\\dagger}$& -0.170 &\\textbf{.007}$^{\\dagger\\dagger}$\\\\\n",
      "coherence\n",
      "coherence & 0.107 & .095& -0.030 & .643& 0.270 &\\textbf{.000}$^{\\dagger\\dagger}$& 0.068 & .286& 0.005 & .935& -0.218 &\\textbf{.001}$^{\\dagger\\dagger}$\\\\\n",
      "clarity\n",
      "clarity & 0.002 & .973& -0.111 & .083& 0.190 &\\textbf{.003}$^{\\dagger\\dagger}$& 0.081 & .204& 0.042 & .514& -0.235 &\\textbf{.000}$^{\\dagger\\dagger}$\\\\\n",
      "learning_effect\n",
      "learning_effect & 0.084 & .192& -0.085 & .184& 0.170 &\\textbf{.008}$^{\\dagger\\dagger}$& -0.006 & .931& 0.007 & .913& -0.135 & .035{\\dagger}$\\\\\n",
      "understanding_of_system\n",
      "understanding_of_system & 0.114 & .076& -0.083 & .197& 0.157 & .014{\\dagger}$& 0.010 & .877& -0.017 & .795& -0.153 & .016{\\dagger}$\\\\\n",
      "usefulness_explanation\n",
      "usefulness_explanation & 0.125 & .051& 0.081 & .206& 0.361 &\\textbf{.000}$^{\\dagger\\dagger}$& 0.266 &\\textbf{.000}$^{\\dagger\\dagger}$& 0.229 &\\textbf{.000}$^{\\dagger\\dagger}$& -0.300 &\\textbf{.000}$^{\\dagger\\dagger}$\\\\\n",
      "objective_feature_understanding\n",
      "objective_feature_understanding & 0.332 &\\textbf{.000}$^{\\dagger\\dagger}$& 0.195 &\\textbf{.002}$^{\\dagger\\dagger}$& 0.469 &\\textbf{.000}$^{\\dagger\\dagger}$& 0.322 &\\textbf{.000}$^{\\dagger\\dagger}$& 0.269 &\\textbf{.000}$^{\\dagger\\dagger}$& -0.297 &\\textbf{.000}$^{\\dagger\\dagger}$\\\\\n",
      "-----------------\n",
      "Reliability/Competence\n",
      "Reliability/Competence & 0.127 & .047{\\dagger}$& 0.090 & .162& 0.224 &\\textbf{.000}$^{\\dagger\\dagger}$& 0.195 &\\textbf{.002}$^{\\dagger\\dagger}$& 0.175 &\\textbf{.006}$^{\\dagger\\dagger}$& -0.200 &\\textbf{.002}$^{\\dagger\\dagger}$\\\\\n",
      "Understandability/Predictability\n",
      "Understandability/Predictability & 0.099 & .123& 0.051 & .430& 0.210 &\\textbf{.001}$^{\\dagger\\dagger}$& 0.132 & .038{\\dagger}$& 0.125 & .051& -0.182 &\\textbf{.004}$^{\\dagger\\dagger}$\\\\\n",
      "Trust in Automation\n",
      "Trust in Automation & 0.145 & .024{\\dagger}$& 0.032 & .617& 0.254 &\\textbf{.000}$^{\\dagger\\dagger}$& 0.164 &\\textbf{.010}$^{\\dagger\\dagger}$& 0.152 & .017{\\dagger}$& -0.203 &\\textbf{.001}$^{\\dagger\\dagger}$\\\\\n",
      "-----------------\n"
     ]
    }
   ],
   "source": [
    "from scipy.stats import spearmanr\n",
    "\n",
    "print(\"Order for reliance measures\", reliance_dimensions)\n",
    "str_dict = {}\n",
    "for dv in explanation_understanding_dimensions:\n",
    "    str_dict[dv] = \"{} \".format(dv)\n",
    "    print(dv)\n",
    "    for reliance_measure in reliance_dimensions:\n",
    "        # print(len(understanding_variable_dict[dv]), understanding_variable_dict[dv])\n",
    "        # print(len(variable_dict[reliance_measure]), variable_dict[reliance_measure])\n",
    "        correlation, pvalue = spearmanr(understanding_variable_dict[dv], variable_dict[reliance_measure])\n",
    "        # if pvalue < 0.05 / 4:\n",
    "        #     print(\"Variable {} and variable {} have spearman correlation {:.3f} and pvalue {:.3f}\".format(dv, reliance_measure, correlation, pvalue))\n",
    "        # str_dict[dv] += \"& {} & {}\".format(\"%.3f\"%correlation, (\"%.3f\"%pvalue)[1:])\n",
    "        if pvalue < 0.0125:\n",
    "            str_dict[dv] += \"& {} &\".format(\"%.3f\"%correlation)\n",
    "            str_dict[dv] += \"\\\\textbf{\"\n",
    "            str_dict[dv] += \"{}\".format((\"%.3f\"%pvalue)[1:])\n",
    "            str_dict[dv] += \"}$^{\\dagger\\dagger}$\"\n",
    "        elif pvalue < 0.05:\n",
    "            str_dict[dv] += \"& {} & {}\".format(\"%.3f\"%correlation, (\"%.3f\"%pvalue)[1:])\n",
    "            str_dict[dv] += \"{\\dagger}$\"\n",
    "        else:\n",
    "            str_dict[dv] += \"& {} & {}\".format(\"%.3f\"%correlation, (\"%.3f\"%pvalue)[1:])\n",
    "    str_dict[dv] += \"\\\\\\\\\"\n",
    "    print(str_dict[dv])\n",
    "print(\"-\" * 17)\n",
    "\n",
    "for dv in trust_dimensions:\n",
    "    str_dict[dv] = \"{} \".format(dv)\n",
    "    print(dv)\n",
    "    for reliance_measure in reliance_dimensions:\n",
    "        correlation, pvalue = spearmanr(variable_dict[dv], variable_dict[reliance_measure])\n",
    "        # if pvalue < 0.05 / 4:\n",
    "        #     print(\"Variable {} and variable {} have spearman correlation {:.3f} and pvalue {:.3f}\".format(dv, reliance_measure, correlation, pvalue))\n",
    "        # print(pvalue)\n",
    "        if pvalue < 0.0125:\n",
    "            str_dict[dv] += \"& {} &\".format(\"%.3f\"%correlation)\n",
    "            str_dict[dv] += \"\\\\textbf{\"\n",
    "            str_dict[dv] += \"{}\".format((\"%.3f\"%pvalue)[1:])\n",
    "            str_dict[dv] += \"}$^{\\dagger\\dagger}$\"\n",
    "        elif pvalue < 0.05:\n",
    "            str_dict[dv] += \"& {} & {}\".format(\"%.3f\"%correlation, (\"%.3f\"%pvalue)[1:])\n",
    "            str_dict[dv] += \"{\\dagger}$\"\n",
    "        else:\n",
    "            str_dict[dv] += \"& {} & {}\".format(\"%.3f\"%correlation, (\"%.3f\"%pvalue)[1:])\n",
    "    str_dict[dv] += \"\\\\\\\\\"\n",
    "    print(str_dict[dv])\n",
    "print(\"-\" * 17)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "agreement_phi",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
