{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from util import load_user_data\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy.stats import kruskal, mannwhitneyu\n",
    "\n",
    "def post_hoc_comparison(data_list_1, data_list_2, name1, name2):\n",
    "\tprint(\"Use pots-hoc analysis\")\n",
    "\tthreshold = 0.05 / 4\n",
    "\tflag = False\n",
    "\tstatistic, pvalue = mannwhitneyu(data_list_1, data_list_2, alternative='greater')\n",
    "\tif pvalue < threshold:\n",
    "\t\tprint(\"Alternative {} > {},\".format(name1, name2), \"pvalue %.4f\"%pvalue, \"statistic %.4f\"%statistic)\n",
    "\t\tflag = True\n",
    "\tstatistic, pvalue = mannwhitneyu(data_list_1, data_list_2, alternative='less')\n",
    "\tif pvalue < threshold:\n",
    "\t\tprint(\"Alternative {} < {},\".format(name1, name2), \"pvalue %.4f\"%pvalue, \"statistic %.4f\"%statistic)\n",
    "\t\tflag = True\n",
    "\tif not flag:\n",
    "\t\tprint(\"No significant difference with post-hoc analysis\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "245 valid participants\n",
      "{'control': 61, 'dashboard': 61, 'chatxai': 62, 'chatxaiboost': 61, 'chatxaiAuto': 0}\n",
      "91 participants blindly rely on AI advice\n",
      "{'control': 8, 'dashboard': 27, 'chatxai': 32, 'chatxaiboost': 24, 'chatxaiAuto': 0}\n",
      "61 valid participants\n",
      "{'control': 0, 'dashboard': 0, 'chatxai': 0, 'chatxaiboost': 0, 'chatxaiAuto': 61}\n",
      "25 participants blindly rely on AI advice\n",
      "{'control': 0, 'dashboard': 0, 'chatxai': 0, 'chatxaiboost': 0, 'chatxaiAuto': 25}\n",
      "----------------------------------\n"
     ]
    }
   ],
   "source": [
    "valid_users, tp_data, _ = load_user_data(filename=\"../data/xailabdata_all.csv\")\n",
    "valid_users_2, tp_data_2, _ = load_user_data(filename=\"../data/xailabdata_llm_agent.csv\")\n",
    "\n",
    "valid_users = valid_users | valid_users_2\n",
    "tp_data.update(tp_data_2)\n",
    "variable_dict = {}\n",
    "variable_dict[\"condition\"] = []\n",
    "dimensions = [\"accuracy\", \"agreement_fraction\", \"switch_fraction\", \"accuracy-wid\", \"RAIR\", \"RSR\"]\n",
    "all_conditions = [\"control\", \"dashboard\", \"chatxai\", \"chatxaiboost\", \"chatxaiAuto\"]\n",
    "for condition in all_conditions:\n",
    "    variable_dict[condition] = {}\n",
    "    for dimension in dimensions:\n",
    "        variable_dict[condition][dimension] = []\n",
    "\n",
    "for user in valid_users:\n",
    "    tp_condition = tp_data[user][\"condition\"]\n",
    "    reliance_measures = tp_data[user][\"reliance_measures\"]\n",
    "    for dimension in dimensions:\n",
    "        variable_dict[tp_condition][dimension].append(reliance_measures[dimension])\n",
    "\n",
    "print(\"-\" * 34)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For all participants, compare with experimental conditions\n",
      "accuracy\n",
      "kruskal test result: H:9.09, p:0.059\n",
      "61, Mean: M(control):0.62, SD(control):0.13\n",
      "61, Mean: M(dashboard):0.65, SD(dashboard):0.11\n",
      "62, Mean: M(chatxai):0.67, SD(chatxai):0.10\n",
      "61, Mean: M(chatxaiboost):0.64, SD(chatxaiboost):0.09\n",
      "61, Mean: M(chatxaiAuto):0.63, SD(chatxaiAuto):0.10\n",
      "accuracy&9.09 & 0.059& $0.62 \\pm 0.13$ &$0.65 \\pm 0.11$ &$0.67 \\pm 0.10$ &$0.64 \\pm 0.09$ &$0.63 \\pm 0.10$ &\n",
      "-----------------\n",
      "agreement_fraction\n",
      "kruskal test result: H:33.66, p:0.000\n",
      "61, Mean: M(control):0.74, SD(control):0.17\n",
      "61, Mean: M(dashboard):0.86, SD(dashboard):0.17\n",
      "62, Mean: M(chatxai):0.89, SD(chatxai):0.15\n",
      "61, Mean: M(chatxaiboost):0.85, SD(chatxaiboost):0.16\n",
      "61, Mean: M(chatxaiAuto):0.89, SD(chatxaiAuto):0.11\n",
      "agreement_fraction&33.66 & 0.000& $0.74 \\pm 0.17$ &$0.86 \\pm 0.17$ &$0.89 \\pm 0.15$ &$0.85 \\pm 0.16$ &$0.89 \\pm 0.11$ &\n",
      "Use pots-hoc analysis\n",
      "Alternative control < dashboard, pvalue 0.0000 statistic 1088.0000\n",
      "Use pots-hoc analysis\n",
      "Alternative control < chatxai, pvalue 0.0000 statistic 969.0000\n",
      "Use pots-hoc analysis\n",
      "Alternative control < chatxaiboost, pvalue 0.0001 statistic 1144.5000\n",
      "Use pots-hoc analysis\n",
      "Alternative control < chatxaiAuto, pvalue 0.0000 statistic 902.5000\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "-----------------\n",
      "switch_fraction\n",
      "kruskal test result: H:19.14, p:0.001\n",
      "61, Mean: M(control):0.31, SD(control):0.34\n",
      "61, Mean: M(dashboard):0.57, SD(dashboard):0.41\n",
      "62, Mean: M(chatxai):0.58, SD(chatxai):0.43\n",
      "61, Mean: M(chatxaiboost):0.57, SD(chatxaiboost):0.41\n",
      "61, Mean: M(chatxaiAuto):0.57, SD(chatxaiAuto):0.41\n",
      "switch_fraction&19.14 & 0.001& $0.31 \\pm 0.34$ &$0.57 \\pm 0.41$ &$0.58 \\pm 0.43$ &$0.57 \\pm 0.41$ &$0.57 \\pm 0.41$ &\n",
      "Use pots-hoc analysis\n",
      "Alternative control < dashboard, pvalue 0.0004 statistic 1220.5000\n",
      "Use pots-hoc analysis\n",
      "Alternative control < chatxai, pvalue 0.0003 statistic 1232.5000\n",
      "Use pots-hoc analysis\n",
      "Alternative control < chatxaiboost, pvalue 0.0003 statistic 1214.5000\n",
      "Use pots-hoc analysis\n",
      "Alternative control < chatxaiAuto, pvalue 0.0002 statistic 1198.0000\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "-----------------\n",
      "accuracy-wid\n",
      "kruskal test result: H:5.06, p:0.281\n",
      "61, Mean: M(control):0.46, SD(control):0.30\n",
      "61, Mean: M(dashboard):0.50, SD(dashboard):0.36\n",
      "62, Mean: M(chatxai):0.52, SD(chatxai):0.35\n",
      "61, Mean: M(chatxaiboost):0.55, SD(chatxaiboost):0.38\n",
      "61, Mean: M(chatxaiAuto):0.42, SD(chatxaiAuto):0.36\n",
      "accuracy-wid&5.06 & 0.281& $0.46 \\pm 0.30$ &$0.50 \\pm 0.36$ &$0.52 \\pm 0.35$ &$0.55 \\pm 0.38$ &$0.42 \\pm 0.36$ &\n",
      "-----------------\n",
      "RAIR\n",
      "kruskal test result: H:11.01, p:0.026\n",
      "61, Mean: M(control):0.35, SD(control):0.39\n",
      "61, Mean: M(dashboard):0.50, SD(dashboard):0.44\n",
      "62, Mean: M(chatxai):0.60, SD(chatxai):0.45\n",
      "61, Mean: M(chatxaiboost):0.52, SD(chatxaiboost):0.44\n",
      "61, Mean: M(chatxaiAuto):0.48, SD(chatxaiAuto):0.45\n",
      "RAIR&11.01 & 0.026& $0.35 \\pm 0.39$ &$0.50 \\pm 0.44$ &$0.60 \\pm 0.45$ &$0.52 \\pm 0.44$ &$0.48 \\pm 0.45$ &\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "Alternative control < chatxai, pvalue 0.0007 statistic 1291.0000\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "-----------------\n",
      "RSR\n",
      "kruskal test result: H:38.26, p:0.000\n",
      "61, Mean: M(control):0.57, SD(control):0.46\n",
      "61, Mean: M(dashboard):0.29, SD(dashboard):0.44\n",
      "62, Mean: M(chatxai):0.23, SD(chatxai):0.40\n",
      "61, Mean: M(chatxaiboost):0.26, SD(chatxaiboost):0.41\n",
      "61, Mean: M(chatxaiAuto):0.11, SD(chatxaiAuto):0.29\n",
      "RSR&38.26 & 0.000& $0.57 \\pm 0.46$ &$0.29 \\pm 0.44$ &$0.23 \\pm 0.40$ &$0.26 \\pm 0.41$ &$0.11 \\pm 0.29$ &\n",
      "Use pots-hoc analysis\n",
      "Alternative control > dashboard, pvalue 0.0003 statistic 2453.0000\n",
      "Use pots-hoc analysis\n",
      "Alternative control > chatxai, pvalue 0.0000 statistic 2626.0000\n",
      "Use pots-hoc analysis\n",
      "Alternative control > chatxaiboost, pvalue 0.0001 statistic 2515.0000\n",
      "Use pots-hoc analysis\n",
      "Alternative control > chatxaiAuto, pvalue 0.0000 statistic 2817.5000\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "Alternative dashboard > chatxaiAuto, pvalue 0.0108 statistic 2190.0000\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "-----------------\n"
     ]
    }
   ],
   "source": [
    "print(\"For all participants, compare with experimental conditions\")\n",
    "for dimension in dimensions:\n",
    "    print(dimension)\n",
    "    kwargs = [variable_dict[condition][dimension] for condition in all_conditions]\n",
    "    statistic, pvalue = kruskal(*kwargs)\n",
    "    print(\"kruskal test result: H:{:.2f}, p:{:.3f}\".format(statistic, pvalue))\n",
    "    tp_str = dimension + \"&\" + \"{:.2f} & {:.3f}& \".format(statistic, pvalue)\n",
    "    for condition in all_conditions:\n",
    "        data_list_1 = variable_dict[condition][dimension]\n",
    "        print(\"{}, Mean: M({}):{:.2f}, SD({}):{:.2f}\".format(len(data_list_1), condition, np.mean(data_list_1), condition, np.std(data_list_1)))\n",
    "        tp_str += \"${:.2f} \\\\pm {:.2f}$ &\".format(np.mean(data_list_1), np.std(data_list_1))\n",
    "    print(tp_str)\n",
    "    # In the prior test, shall we use 0.05 and then reach a conclusion with calibrated threshold?\n",
    "    if pvalue < 0.05:\n",
    "        length = len(all_conditions)\n",
    "        for i in range(length - 1):\n",
    "            for j in range(i+1, length):\n",
    "                group_1 = all_conditions[i]\n",
    "                group_2 = all_conditions[j]\n",
    "                data_list_1 = variable_dict[group_1][dimension]\n",
    "                data_list_2 = variable_dict[group_2][dimension]\n",
    "                post_hoc_comparison(data_list_1, data_list_2, group_1, group_2)\n",
    "    print(\"-\" * 17)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "agreement_phi",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
