{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "245 valid participants\n",
      "{'control': 61, 'dashboard': 61, 'chatxai': 62, 'chatxaiboost': 61, 'chatxaiAuto': 0}\n",
      "91 participants blindly rely on AI advice\n",
      "{'control': 8, 'dashboard': 27, 'chatxai': 32, 'chatxaiboost': 24, 'chatxaiAuto': 0}\n",
      "61 valid participants\n",
      "{'control': 0, 'dashboard': 0, 'chatxai': 0, 'chatxaiboost': 0, 'chatxaiAuto': 61}\n",
      "25 participants blindly rely on AI advice\n",
      "{'control': 0, 'dashboard': 0, 'chatxai': 0, 'chatxaiboost': 0, 'chatxaiAuto': 25}\n"
     ]
    }
   ],
   "source": [
    "from util import load_user_data\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import sys\n",
    "valid_users, tp_data, _ = load_user_data(filename=\"../data/xailabdata_all.csv\")\n",
    "valid_users_2, tp_data_2, _ = load_user_data(filename=\"../data/xailabdata_llm_agent.csv\")\n",
    "\n",
    "valid_users = valid_users | valid_users_2\n",
    "tp_data.update(tp_data_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "778\n",
      "157\n"
     ]
    }
   ],
   "source": [
    "explainers = [\"pdp\", \"shap\", \"whatIf\", \"decisionTree\", \"counterFactual\"]\n",
    "condition_usage = {}\n",
    "for condition in [\"control\", \"dashboard\", \"chatxai\", \"chatxaiboost\", \"chatxaiAuto\"]:\n",
    "    condition_usage[condition] = {}\n",
    "    for explainer in explainers:\n",
    "        condition_usage[condition][explainer] = []\n",
    "\n",
    "limited_interaction = 0\n",
    "limited_users = set()\n",
    "for user in valid_users:\n",
    "    # print(tp_data[user].keys())\n",
    "    tp_condition = tp_data[user][\"condition\"]\n",
    "    if tp_condition in [\"control\", \"dashboard\"]:\n",
    "        continue\n",
    "    # print(tp_condition)\n",
    "    # print(tp_data[user].keys())\n",
    "    task_order = tp_data[user][\"task_order\"]\n",
    "    xai_usage = tp_data[user][\"xai_usage\"]\n",
    "    tp_condition_usage = {}\n",
    "    for explainer in explainers:\n",
    "        tp_condition_usage[explainer] = 0\n",
    "    for task_id in task_order:\n",
    "        # print(xai_usage)\n",
    "        if task_id not in xai_usage:\n",
    "            print(xai_usage, task_id)\n",
    "            sys.exit(-1)\n",
    "        metadata = xai_usage[task_id][\"explainer_counter\"]\n",
    "        total_usage_tp = 0\n",
    "        # print(tp_condition, metadata)\n",
    "        for explainer in explainers:\n",
    "            assert isinstance(metadata[explainer], int)\n",
    "            # condition_usage[tp_condition][explainer].append(metadata[explainer])\n",
    "            tp_condition_usage[explainer] += metadata[explainer]\n",
    "            total_usage_tp += metadata[explainer]\n",
    "        if total_usage_tp <= 2:\n",
    "            limited_interaction += 1\n",
    "            limited_users.add(user)\n",
    "    for explainer in explainers:\n",
    "        condition_usage[tp_condition][explainer].append(tp_condition_usage[explainer])\n",
    "print(limited_interaction)\n",
    "print(len(limited_users))\n",
    "# sys.exit(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pdp\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "Alternative CXAI > LLM Agent, pvalue 0.0000 statistic 3442.0000\n",
      "Use pots-hoc analysis\n",
      "Alternative PCXAI > LLM Agent, pvalue 0.0000 statistic 3287.5000\n",
      "62 61 61\n",
      "kruskal test result: H:76.85, p:0.000\n",
      "On average: M(CXAI):13.5 M(PCXAI):14.1 M(LLM Agent):3.6\n",
      "SD(CXAI):7.3 SD(PCXAI):8.5 SD(LLM Agent):3.5\n",
      "-----------------\n",
      "shap\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "Alternative CXAI < LLM Agent, pvalue 0.0001 statistic 1169.5000\n",
      "Use pots-hoc analysis\n",
      "Alternative PCXAI < LLM Agent, pvalue 0.0059 statistic 1373.5000\n",
      "62 61 61\n",
      "kruskal test result: H:15.32, p:0.000\n",
      "On average: M(CXAI):6.7 M(PCXAI):7.8 M(LLM Agent):9.5\n",
      "SD(CXAI):3.7 SD(PCXAI):3.3 SD(LLM Agent):4.3\n",
      "-----------------\n",
      "whatIf\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "Alternative CXAI < LLM Agent, pvalue 0.0009 statistic 1274.5000\n",
      "Use pots-hoc analysis\n",
      "Alternative PCXAI < LLM Agent, pvalue 0.0001 statistic 1133.0000\n",
      "62 61 61\n",
      "kruskal test result: H:16.08, p:0.000\n",
      "On average: M(CXAI):4.1 M(PCXAI):3.6 M(LLM Agent):7.1\n",
      "SD(CXAI):4.2 SD(PCXAI):3.4 SD(LLM Agent):5.9\n",
      "-----------------\n",
      "decisionTree\n",
      "62 61 61\n",
      "kruskal test result: H:8.48, p:0.014\n",
      "On average: M(CXAI):2.8 M(PCXAI):3.1 M(LLM Agent):4.8\n",
      "SD(CXAI):3.0 SD(PCXAI):3.3 SD(LLM Agent):3.9\n",
      "-----------------\n",
      "counterFactual\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "Alternative CXAI < LLM Agent, pvalue 0.0003 statistic 1223.5000\n",
      "Use pots-hoc analysis\n",
      "Alternative PCXAI < LLM Agent, pvalue 0.0002 statistic 1181.5000\n",
      "62 61 61\n",
      "kruskal test result: H:16.25, p:0.000\n",
      "On average: M(CXAI):1.8 M(PCXAI):1.8 M(LLM Agent):3.7\n",
      "SD(CXAI):1.9 SD(PCXAI):1.9 SD(LLM Agent):3.1\n",
      "-----------------\n"
     ]
    }
   ],
   "source": [
    "from scipy.stats import kruskal, mannwhitneyu\n",
    "def post_hoc_comparison(data_list_1, data_list_2, name1, name2):\n",
    "\tprint(\"Use pots-hoc analysis\")\n",
    "\tthreshold = 0.05 / 4\n",
    "\tflag = False\n",
    "\tstatistic, pvalue = mannwhitneyu(data_list_1, data_list_2, alternative='greater')\n",
    "\tif pvalue < threshold:\n",
    "\t\tprint(\"Alternative {} > {},\".format(name1, name2), \"pvalue %.4f\"%pvalue, \"statistic %.4f\"%statistic)\n",
    "\t\tflag = True\n",
    "\tstatistic, pvalue = mannwhitneyu(data_list_1, data_list_2, alternative='less')\n",
    "\tif pvalue < threshold:\n",
    "\t\tprint(\"Alternative {} < {},\".format(name1, name2), \"pvalue %.4f\"%pvalue, \"statistic %.4f\"%statistic)\n",
    "\t\tflag = True\n",
    "\tif not flag:\n",
    "\t\tprint(\"No significant difference with post-hoc analysis\")\n",
    "\n",
    "for explainer in explainers:\n",
    "    data_list_1 = condition_usage[\"chatxai\"][explainer]\n",
    "    data_list_2 = condition_usage[\"chatxaiboost\"][explainer]\n",
    "    data_list_3 = condition_usage[\"chatxaiAuto\"][explainer]\n",
    "    statistic, pvalue = kruskal(data_list_1, data_list_2, data_list_3)\n",
    "    print(explainer)\n",
    "    if pvalue < 0.05 / 4:\n",
    "        post_hoc_comparison(data_list_1, data_list_2, \"CXAI\", \"PCXAI\")\n",
    "        post_hoc_comparison(data_list_1, data_list_3, \"CXAI\", \"LLM Agent\")\n",
    "        post_hoc_comparison(data_list_2, data_list_3, \"PCXAI\", \"LLM Agent\")\n",
    "    print(len(data_list_1), len(data_list_2), len(data_list_3))\n",
    "    print(\"kruskal test result: H:{:.2f}, p:{:.3f}\".format(statistic, pvalue))\n",
    "    print(\"On average: M(CXAI):{:.1f} M(PCXAI):{:.1f} M(LLM Agent):{:.1f}\".format(np.mean(data_list_1), np.mean(data_list_2), np.mean(data_list_3)))\n",
    "    print(\"SD(CXAI):{:.1f} SD(PCXAI):{:.1f} SD(LLM Agent):{:.1f}\".format(np.std(data_list_1), np.std(data_list_2), np.std(data_list_3)))\n",
    "    print(\"-\" * 17)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dashboard: M: 0.96\n",
      "chatxai: M: 0.93\n",
      "chatxaiboost: M: 0.98\n",
      "chatxaiAuto: M: 0.97\n",
      "kruskal test result: H:0.62, p:0.733\n",
      "On average: M(CXAI):0.93 M(PCXAI):0.98 M(LLM Agent):0.97\n",
      "-----------------\n"
     ]
    }
   ],
   "source": [
    "condition_FS = {}\n",
    "for condition in [\"dashboard\", \"chatxai\", \"chatxaiboost\", \"chatxaiAuto\"]:\n",
    "    condition_FS[condition] = []\n",
    "for user in valid_users:\n",
    "    condition = tp_data[user][\"condition\"]\n",
    "    if condition == \"control\":\n",
    "        continue\n",
    "    feature_switch = tp_data[user][\"explanation_understanding\"][\"feature_switch\"]\n",
    "    condition_FS[condition].append(feature_switch)\n",
    "for condition in [\"dashboard\", \"chatxai\", \"chatxaiboost\", \"chatxaiAuto\"]:\n",
    "    print(\"{}: M: {:.2f}\".format(condition, np.mean(condition_FS[condition])))\n",
    "\n",
    "data_list_1 = condition_FS[\"chatxai\"]\n",
    "data_list_2 = condition_FS[\"chatxaiboost\"]\n",
    "data_list_3 = condition_FS[\"chatxaiAuto\"]\n",
    "statistic, pvalue = kruskal(data_list_1, data_list_2, data_list_3)\n",
    "print(\"kruskal test result: H:{:.2f}, p:{:.3f}\".format(statistic, pvalue))\n",
    "print(\"On average: M(CXAI):{:.2f} M(PCXAI):{:.2f} M(LLM Agent):{:.2f}\".format(np.mean(data_list_1), np.mean(data_list_2), np.mean(data_list_3)))\n",
    "print(\"-\" * 17)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "agreement_phi",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
