{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from util import load_user_data\n",
    "import pandas as pd\n",
    "import sys\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "245 valid participants\n",
      "{'control': 61, 'dashboard': 61, 'chatxai': 62, 'chatxaiboost': 61, 'chatxaiAuto': 0}\n",
      "91 participants blindly rely on AI advice\n",
      "{'control': 8, 'dashboard': 27, 'chatxai': 32, 'chatxaiboost': 24, 'chatxaiAuto': 0}\n",
      "61 valid participants\n",
      "{'control': 0, 'dashboard': 0, 'chatxai': 0, 'chatxaiboost': 0, 'chatxaiAuto': 61}\n",
      "25 participants blindly rely on AI advice\n",
      "{'control': 0, 'dashboard': 0, 'chatxai': 0, 'chatxaiboost': 0, 'chatxaiAuto': 25}\n",
      "306\n"
     ]
    }
   ],
   "source": [
    "valid_users, tp_data, _ = load_user_data(filename=\"../data/xailabdata_all.csv\")\n",
    "valid_users_2, tp_data_2, _ = load_user_data(filename=\"../data/xailabdata_llm_agent.csv\")\n",
    "\n",
    "valid_users = valid_users | valid_users_2\n",
    "tp_data.update(tp_data_2)\n",
    "\n",
    "all_conditions = [\"control\", \"dashboard\", \"chatxai\", \"chatxaiboost\", \"chatxaiAuto\"]\n",
    "condition_confidence_dynamics_before = {}\n",
    "condition_confidence_dynamics_after = {}\n",
    "number_tasks = 10\n",
    "for condition in all_conditions:\n",
    "    condition_confidence_dynamics_after[condition] = []\n",
    "    condition_confidence_dynamics_before[condition] = []\n",
    "    for i in range(number_tasks):\n",
    "        condition_confidence_dynamics_after[condition].append([])\n",
    "        condition_confidence_dynamics_before[condition].append([])\n",
    "condition_confidence_before = {}\n",
    "condition_confidence_after  = {}\n",
    "for condition in all_conditions:\n",
    "    condition_confidence_before[condition] = []\n",
    "    condition_confidence_after[condition]  = []\n",
    "variable_dict = {\n",
    "    \"condition\": [],\n",
    "    \"initial_confidence\": [],\n",
    "    \"final_confidence\": []\n",
    "}\n",
    "user_confidence_vec = {}\n",
    "for user in valid_users:\n",
    "    user_confidence_vec[user] = {}\n",
    "    tp_condition = tp_data[user][\"condition\"]\n",
    "    user_task_order = tp_data[user][\"task_order\"]\n",
    "    user_confidence = tp_data[user][\"confidence\"]\n",
    "    variable_dict[\"condition\"].append(tp_condition)\n",
    "    # print(user_confidence)\n",
    "    try:\n",
    "        assert len(user_task_order) == 10\n",
    "    except:\n",
    "        print(len(user_task_order))\n",
    "    # print(data_long_format[tp_condition])\n",
    "    # print(tp_condition, len(data_long_format[tp_condition][\"confidence\"]))\n",
    "    confidence_before_advice = []\n",
    "    confidence_after_advice = []\n",
    "    for index, task_id in enumerate(user_task_order):\n",
    "        tp_confidence = int(user_confidence[(task_id, \"advice\")])\n",
    "        # assert isinstance(tp_confidence, int)\n",
    "        condition_confidence_dynamics_after[tp_condition][index].append(tp_confidence)\n",
    "        confidence_after_advice.append(tp_confidence)\n",
    "\n",
    "        tp_confidence = int(user_confidence[(task_id, \"base\")])\n",
    "        condition_confidence_dynamics_before[tp_condition][index].append(tp_confidence)\n",
    "        confidence_before_advice.append(tp_confidence)\n",
    "    user_confidence_vec[user][\"initial\"] = confidence_before_advice\n",
    "    user_confidence_vec[user][\"final\"] = confidence_after_advice\n",
    "    condition_confidence_after[tp_condition].append(np.mean(confidence_after_advice))\n",
    "    condition_confidence_before[tp_condition].append(np.mean(confidence_before_advice))\n",
    "    variable_dict[\"initial_confidence\"].append(np.mean(confidence_before_advice))\n",
    "    variable_dict[\"final_confidence\"].append(np.mean(confidence_after_advice))\n",
    "# for condition in all_conditions:\n",
    "#     print(len(condition_confidence_before[condition]), len(condition_confidence_after[condition]))\n",
    "print(len(variable_dict[\"condition\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import kruskal, mannwhitneyu\n",
    "def post_hoc_comparison(data_list_1, data_list_2, name1, name2):\n",
    "\tprint(\"Use pots-hoc analysis\")\n",
    "\tthreshold = 0.05 / 4\n",
    "\tflag = False\n",
    "\tstatistic, pvalue = mannwhitneyu(data_list_1, data_list_2, alternative='greater')\n",
    "\tif pvalue < threshold:\n",
    "\t\tprint(\"Alternative {} > {},\".format(name1, name2), \"pvalue %.4f\"%pvalue, \"statistic %.4f\"%statistic)\n",
    "\t\tflag = True\n",
    "\tstatistic, pvalue = mannwhitneyu(data_list_1, data_list_2, alternative='less')\n",
    "\tif pvalue < threshold:\n",
    "\t\tprint(\"Alternative {} < {},\".format(name1, name2), \"pvalue %.4f\"%pvalue, \"statistic %.4f\"%statistic)\n",
    "\t\tflag = True\n",
    "\tif not flag:\n",
    "\t\tprint(\"No significant difference with post-hoc analysis\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import kstest, ks_2samp, ttest_rel, wasserstein_distance\n",
    "from scipy.spatial.distance import jensenshannon \n",
    "# import jensenshannon, wasserstein_distance\n",
    "import sys\n",
    "condition_dict = {}\n",
    "for condition in all_conditions:\n",
    "    condition_dict[condition] = []\n",
    "for user in valid_users:\n",
    "    tp_condition = tp_data[user][\"condition\"]\n",
    "    confidence_before_advice = [item for item in user_confidence_vec[user][\"initial\"]]\n",
    "    confidence_after_advice  = [item for item in user_confidence_vec[user][\"final\"]]\n",
    "    # res = ks_2samp(confidence_before_advice, confidence_after_advice)\n",
    "    # stat, pvalue = res\n",
    "    # print(stat)\n",
    "    # t_stat, p_value = ttest_rel(confidence_before_advice, confidence_after_advice)\n",
    "    # print(f\"Paired t-test: t-statistic = {t_stat}, p-value = {p_value}\")\n",
    "    # sys.exit(-1)\n",
    "    emd = wasserstein_distance(confidence_before_advice, confidence_after_advice)\n",
    "    # emd = jensenshannon(confidence_before_advice, confidence_after_advice)\n",
    "    # print(f\"Earth Mover's Distance (Wasserstein Distance): {emd}\")\n",
    "    condition_dict[tp_condition].append(emd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kruskal test result: H:5.59, p:0.232\n",
      "61, Mean: M(control):0.35, SD(control):0.32\n",
      "61, Mean: M(dashboard):0.42, SD(dashboard):0.28\n",
      "62, Mean: M(chatxai):0.39, SD(chatxai):0.31\n",
      "61, Mean: M(chatxaiboost):0.44, SD(chatxaiboost):0.31\n",
      "61, Mean: M(chatxaiAuto):0.48, SD(chatxaiAuto):0.40\n",
      "Final&5.59 & 0.232& $0.35 \\pm 0.32$ &$0.42 \\pm 0.28$ &$0.39 \\pm 0.31$ &$0.44 \\pm 0.31$ &$0.48 \\pm 0.40$ &\n",
      "-----------------\n"
     ]
    }
   ],
   "source": [
    "kwargs = [condition_dict[condition] for condition in all_conditions]\n",
    "statistic, pvalue = kruskal(*kwargs)\n",
    "print(\"kruskal test result: H:{:.2f}, p:{:.3f}\".format(statistic, pvalue))\n",
    "tp_str = \"Final\" + \"&\" + \"{:.2f} & {:.3f}& \".format(statistic, pvalue)\n",
    "for condition in all_conditions:\n",
    "    data_list_1 = condition_dict[condition]\n",
    "    print(\"{}, Mean: M({}):{:.2f}, SD({}):{:.2f}\".format(len(data_list_1), condition, np.mean(data_list_1), condition, np.std(data_list_1)))\n",
    "    tp_str += \"${:.2f} \\\\pm {:.2f}$ &\".format(np.mean(data_list_1), np.std(data_list_1))\n",
    "print(tp_str)\n",
    "if pvalue < 0.05 / 4:\n",
    "    length = len(all_conditions)\n",
    "    for i in range(length - 1):\n",
    "        for j in range(i+1, length):\n",
    "            group_1 = all_conditions[i]\n",
    "            group_2 = all_conditions[j]\n",
    "            data_list_1 = condition_dict[group_1]\n",
    "            data_list_2 = condition_dict[group_2]\n",
    "            post_hoc_comparison(data_list_1, data_list_2, group_1, group_2)\n",
    "print(\"-\" * 17)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "initial_confidence\n",
      "      Source  ddof1  ddof2      F  p-unc     n2\n",
      "0  condition      4    302  0.483  0.748  0.006\n",
      "final_confidence\n",
      "      Source  ddof1  ddof2      F  p-unc     n2\n",
      "0  condition      4    302  0.651  0.627  0.009\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ghe/opt/anaconda3/envs/agreement_phi/lib/python3.8/site-packages/outdated/utils.py:14: OutdatedPackageWarning: The package pingouin is out of date. Your version is 0.5.3, the latest is 0.5.4.\n",
      "Set the environment variable OUTDATED_IGNORE=1 to disable these warnings.\n",
      "  return warn(\n"
     ]
    }
   ],
   "source": [
    "df = pd.DataFrame(variable_dict)\n",
    "\n",
    "from pingouin import ancova, anova\n",
    "from statsmodels.stats.multicomp import pairwise_tukeyhsd\n",
    "for dimension in [\"initial_confidence\", \"final_confidence\"]:\n",
    "    print(dimension)\n",
    "    aov = anova(dv=dimension, between='condition', data=df, effsize='n2')\n",
    "    print(aov.round(3))\n",
    "    if aov.to_dict()['p-unc'][0] <= 0.05 / 4:\n",
    "        tukey = pairwise_tukeyhsd(endog=df[dimension], groups=df['condition'], alpha=0.0125)\n",
    "        print(tukey)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kruskal test result: H:4.50, p:0.342\n",
      "61, Mean: M(control):3.86, SD(control):0.71\n",
      "61, Mean: M(dashboard):3.85, SD(dashboard):0.54\n",
      "62, Mean: M(chatxai):4.00, SD(chatxai):0.68\n",
      "61, Mean: M(chatxaiboost):3.91, SD(chatxaiboost):0.66\n",
      "62, Mean: M(chatxaiAuto):3.93, SD(chatxaiAuto):0.75\n",
      "Initial&4.50 & 0.342& $3.86 \\pm 0.71$ &$3.85 \\pm 0.54$ &$4.00 \\pm 0.68$ &$3.91 \\pm 0.66$ &$3.93 \\pm 0.75$ &\n",
      "-----------------\n"
     ]
    }
   ],
   "source": [
    "kwargs = [condition_confidence_before[condition] for condition in all_conditions]\n",
    "statistic, pvalue = kruskal(*kwargs)\n",
    "print(\"kruskal test result: H:{:.2f}, p:{:.3f}\".format(statistic, pvalue))\n",
    "tp_str = \"Initial\" + \"&\" + \"{:.2f} & {:.3f}& \".format(statistic, pvalue)\n",
    "for condition in all_conditions:\n",
    "    data_list_1 = condition_confidence_before[condition]\n",
    "    print(\"{}, Mean: M({}):{:.2f}, SD({}):{:.2f}\".format(len(data_list_1), condition, np.mean(data_list_1), condition, np.std(data_list_1)))\n",
    "    tp_str += \"${:.2f} \\\\pm {:.2f}$ &\".format(np.mean(data_list_1), np.std(data_list_1))\n",
    "print(tp_str)\n",
    "if pvalue < 0.05 / 4:\n",
    "    length = len(all_conditions)\n",
    "    for i in range(length - 1):\n",
    "        for j in range(i+1, length):\n",
    "            group_1 = all_conditions[i]\n",
    "            group_2 = all_conditions[j]\n",
    "            data_list_1 = condition_confidence_before[group_1]\n",
    "            data_list_2 = condition_confidence_before[group_2]\n",
    "            post_hoc_comparison(data_list_1, data_list_2, group_1, group_2)\n",
    "print(\"-\" * 17)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kruskal test result: H:4.57, p:0.334\n",
      "61, Mean: M(control):4.07, SD(control):0.63\n",
      "61, Mean: M(dashboard):4.04, SD(dashboard):0.52\n",
      "62, Mean: M(chatxai):4.13, SD(chatxai):0.64\n",
      "61, Mean: M(chatxaiboost):4.12, SD(chatxaiboost):0.58\n",
      "62, Mean: M(chatxaiAuto):4.20, SD(chatxaiAuto):0.63\n",
      "Final&4.57 & 0.334& $4.07 \\pm 0.63$ &$4.04 \\pm 0.52$ &$4.13 \\pm 0.64$ &$4.12 \\pm 0.58$ &$4.20 \\pm 0.63$ &\n",
      "-----------------\n"
     ]
    }
   ],
   "source": [
    "kwargs = [condition_confidence_after[condition] for condition in all_conditions]\n",
    "statistic, pvalue = kruskal(*kwargs)\n",
    "print(\"kruskal test result: H:{:.2f}, p:{:.3f}\".format(statistic, pvalue))\n",
    "tp_str = \"Final\" + \"&\" + \"{:.2f} & {:.3f}& \".format(statistic, pvalue)\n",
    "for condition in all_conditions:\n",
    "    data_list_1 = condition_confidence_after[condition]\n",
    "    print(\"{}, Mean: M({}):{:.2f}, SD({}):{:.2f}\".format(len(data_list_1), condition, np.mean(data_list_1), condition, np.std(data_list_1)))\n",
    "    tp_str += \"${:.2f} \\\\pm {:.2f}$ &\".format(np.mean(data_list_1), np.std(data_list_1))\n",
    "print(tp_str)\n",
    "if pvalue < 0.05 / 4:\n",
    "    length = len(all_conditions)\n",
    "    for i in range(length - 1):\n",
    "        for j in range(i+1, length):\n",
    "            group_1 = all_conditions[i]\n",
    "            group_2 = all_conditions[j]\n",
    "            data_list_1 = condition_confidence_after[group_1]\n",
    "            data_list_2 = condition_confidence_after[group_2]\n",
    "            post_hoc_comparison(data_list_1, data_list_2, group_1, group_2)\n",
    "print(\"-\" * 17)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "agreement_phi",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
