{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from util import load_user_data\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import kruskal, mannwhitneyu\n",
    "def post_hoc_comparison(data_list_1, data_list_2, name1, name2):\n",
    "\tprint(\"Use pots-hoc analysis\")\n",
    "\tthreshold = 0.05 / 4\n",
    "\tflag = False\n",
    "\tstatistic, pvalue = mannwhitneyu(data_list_1, data_list_2, alternative='greater')\n",
    "\tif pvalue < threshold:\n",
    "\t\tprint(\"Alternative {} > {},\".format(name1, name2), \"pvalue %.4f\"%pvalue, \"statistic %.4f\"%statistic)\n",
    "\t\tflag = True\n",
    "\tstatistic, pvalue = mannwhitneyu(data_list_1, data_list_2, alternative='less')\n",
    "\tif pvalue < threshold:\n",
    "\t\tprint(\"Alternative {} < {},\".format(name1, name2), \"pvalue %.4f\"%pvalue, \"statistic %.4f\"%statistic)\n",
    "\t\tflag = True\n",
    "\tif not flag:\n",
    "\t\tprint(\"No significant difference with post-hoc analysis\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "245 valid participants\n",
      "{'control': 61, 'dashboard': 61, 'chatxai': 62, 'chatxaiboost': 61, 'chatxaiAuto': 0}\n",
      "91 participants blindly rely on AI advice\n",
      "{'control': 8, 'dashboard': 27, 'chatxai': 32, 'chatxaiboost': 24, 'chatxaiAuto': 0}\n",
      "61 valid participants\n",
      "{'control': 0, 'dashboard': 0, 'chatxai': 0, 'chatxaiboost': 0, 'chatxaiAuto': 61}\n",
      "25 participants blindly rely on AI advice\n",
      "{'control': 0, 'dashboard': 0, 'chatxai': 0, 'chatxaiboost': 0, 'chatxaiAuto': 25}\n",
      "(245, 13)\n"
     ]
    }
   ],
   "source": [
    "valid_users, tp_data, _ = load_user_data(filename=\"../data/xailabdata_all.csv\")\n",
    "valid_users_2, tp_data_2, _ = load_user_data(filename=\"../data/xailabdata_llm_agent.csv\")\n",
    "\n",
    "valid_users = valid_users | valid_users_2\n",
    "tp_data.update(tp_data_2)\n",
    "variable_dict = {}\n",
    "variable_dict[\"condition\"] = []\n",
    "dimensions = [\"perceived_feature_understanding\", \"completeness\", \"coherence\", \"clarity\", \\\n",
    "    \"learning_effect\", \"understanding_of_system\", \"usefulness_explanation\", \"objective_feature_understanding\"]\n",
    "for dimension in dimensions:\n",
    "    variable_dict[dimension] = []\n",
    "for dimension in [\"Propensity to Trust\", \"Familiarity\", \"ATI\", \"mlbackground\"]:\n",
    "    variable_dict[dimension] = []\n",
    "for user in valid_users:\n",
    "    tp_condition = tp_data[user][\"condition\"]\n",
    "    if tp_condition == \"control\":\n",
    "        continue\n",
    "    trust = tp_data[user][\"Trust_in_automation\"]\n",
    "    explanation_understanding = tp_data[user][\"explanation_understanding\"]\n",
    "    variable_dict[\"condition\"].append(tp_condition)\n",
    "    for dimension in dimensions:\n",
    "        variable_dict[dimension].append(explanation_understanding[dimension])\n",
    "    for dimension in [\"Propensity to Trust\", \"Familiarity\"]:\n",
    "        variable_dict[dimension].append(trust[dimension])\n",
    "    for dimension in [\"ATI\", \"mlbackground\"]:\n",
    "        variable_dict[dimension].append(tp_data[user][dimension])\n",
    "\n",
    "df = pd.DataFrame(variable_dict)\n",
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------\n",
      "For all participants, compare with experimental conditions\n",
      "perceived_feature_understanding\n",
      "      Source  ddof1  ddof2      F  p-unc     n2\n",
      "0  condition      3    241  0.585  0.625  0.007\n",
      "condition\n",
      "chatxai         4.24\n",
      "chatxaiAuto     4.08\n",
      "chatxaiboost    4.07\n",
      "dashboard       4.10\n",
      "Name: perceived_feature_understanding, dtype: float64\n",
      "condition\n",
      "chatxai         0.72\n",
      "chatxaiAuto     0.92\n",
      "chatxaiboost    0.79\n",
      "dashboard       0.89\n",
      "Name: perceived_feature_understanding, dtype: float64\n",
      "-----------------\n",
      "completeness\n",
      "      Source  ddof1  ddof2      F  p-unc     n2\n",
      "0  condition      3    241  0.131  0.942  0.002\n",
      "condition\n",
      "chatxai         3.56\n",
      "chatxaiAuto     3.60\n",
      "chatxaiboost    3.53\n",
      "dashboard       3.60\n",
      "Name: completeness, dtype: float64\n",
      "condition\n",
      "chatxai         0.72\n",
      "chatxaiAuto     0.77\n",
      "chatxaiboost    0.64\n",
      "dashboard       0.67\n",
      "Name: completeness, dtype: float64\n",
      "-----------------\n",
      "coherence\n",
      "      Source  ddof1  ddof2      F  p-unc     n2\n",
      "0  condition      3    241  1.188  0.315  0.015\n",
      "condition\n",
      "chatxai         3.81\n",
      "chatxaiAuto     3.66\n",
      "chatxaiboost    3.87\n",
      "dashboard       3.57\n",
      "Name: coherence, dtype: float64\n",
      "condition\n",
      "chatxai         0.92\n",
      "chatxaiAuto     1.00\n",
      "chatxaiboost    0.99\n",
      "dashboard       0.97\n",
      "Name: coherence, dtype: float64\n",
      "-----------------\n",
      "clarity\n",
      "      Source  ddof1  ddof2      F  p-unc   n2\n",
      "0  condition      3    241  0.025  0.995  0.0\n",
      "condition\n",
      "chatxai         4.02\n",
      "chatxaiAuto     3.98\n",
      "chatxaiboost    4.02\n",
      "dashboard       4.02\n",
      "Name: clarity, dtype: float64\n",
      "condition\n",
      "chatxai         0.76\n",
      "chatxaiAuto     0.83\n",
      "chatxaiboost    0.83\n",
      "dashboard       0.83\n",
      "Name: clarity, dtype: float64\n",
      "-----------------\n",
      "learning_effect\n",
      "      Source  ddof1  ddof2      F  p-unc     n2\n",
      "0  condition      3    241  1.324  0.267  0.016\n",
      "condition\n",
      "chatxai         4.06\n",
      "chatxaiAuto     3.79\n",
      "chatxaiboost    4.03\n",
      "dashboard       3.93\n",
      "Name: learning_effect, dtype: float64\n",
      "condition\n",
      "chatxai         0.77\n",
      "chatxaiAuto     1.00\n",
      "chatxaiboost    0.73\n",
      "dashboard       0.87\n",
      "Name: learning_effect, dtype: float64\n",
      "-----------------\n",
      "understanding_of_system\n",
      "      Source  ddof1  ddof2      F  p-unc     n2\n",
      "0  condition      3    241  0.623  0.601  0.008\n",
      "condition\n",
      "chatxai         3.95\n",
      "chatxaiAuto     3.93\n",
      "chatxaiboost    4.03\n",
      "dashboard       4.13\n",
      "Name: understanding_of_system, dtype: float64\n",
      "condition\n",
      "chatxai         0.86\n",
      "chatxaiAuto     1.03\n",
      "chatxaiboost    0.87\n",
      "dashboard       0.78\n",
      "Name: understanding_of_system, dtype: float64\n",
      "-----------------\n",
      "usefulness_explanation\n",
      "      Source  ddof1  ddof2      F  p-unc     n2\n",
      "0  condition      3    241  0.447  0.719  0.006\n",
      "condition\n",
      "chatxai         4.09\n",
      "chatxaiAuto     3.99\n",
      "chatxaiboost    4.02\n",
      "dashboard       3.95\n",
      "Name: usefulness_explanation, dtype: float64\n",
      "condition\n",
      "chatxai         0.67\n",
      "chatxaiAuto     0.77\n",
      "chatxaiboost    0.59\n",
      "dashboard       0.75\n",
      "Name: usefulness_explanation, dtype: float64\n",
      "-----------------\n",
      "objective_feature_understanding\n",
      "      Source  ddof1  ddof2      F  p-unc     n2\n",
      "0  condition      3    241  3.892   0.01  0.046\n",
      "condition\n",
      "chatxai         0.88\n",
      "chatxaiAuto     0.84\n",
      "chatxaiboost    0.87\n",
      "dashboard       0.88\n",
      "Name: objective_feature_understanding, dtype: float64\n",
      "condition\n",
      "chatxai         0.08\n",
      "chatxaiAuto     0.06\n",
      "chatxaiboost    0.09\n",
      "dashboard       0.07\n",
      "Name: objective_feature_understanding, dtype: float64\n",
      "-----------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ghe/opt/anaconda3/envs/agreement_phi/lib/python3.8/site-packages/outdated/utils.py:14: OutdatedPackageWarning: The package pingouin is out of date. Your version is 0.5.3, the latest is 0.5.5.\n",
      "Set the environment variable OUTDATED_IGNORE=1 to disable these warnings.\n",
      "  return warn(\n"
     ]
    }
   ],
   "source": [
    "print(\"-\" * 34)\n",
    "from pingouin import ancova, anova\n",
    "print(\"For all participants, compare with experimental conditions\")\n",
    "for dimension in dimensions:\n",
    "    print(dimension)\n",
    "    aov = anova(dv=dimension, between='condition', data=df, effsize='n2')\n",
    "    print(aov.round(3))\n",
    "    print(df.groupby(['condition'])[dimension].mean().round(2))\n",
    "    print(df.groupby(['condition'])[dimension].std().round(2))\n",
    "    print(\"-\" * 17)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------\n",
      "For all participants, compare with experimental conditions\n",
      "perceived_feature_understanding\n",
      "                Source       SS   DF       F  p-unc     n2\n",
      "0            condition    1.074    3   0.610  0.609  0.007\n",
      "1  Propensity to Trust   23.590    1  40.150  0.000  0.144\n",
      "2          Familiarity    0.172    1   0.293  0.589  0.001\n",
      "3                  ATI    0.192    1   0.327  0.568  0.001\n",
      "4         mlbackground    0.053    1   0.089  0.765  0.000\n",
      "5             Residual  139.247  237     NaN    NaN    NaN\n",
      "0.61 & .609 & 40.15 & .000 & 0.29 & .589 & 0.33 & .568 & 0.09 & .765\\\\\n",
      "condition\n",
      "chatxai         4.24\n",
      "chatxaiAuto     4.08\n",
      "chatxaiboost    4.07\n",
      "dashboard       4.10\n",
      "Name: perceived_feature_understanding, dtype: float64\n",
      "condition\n",
      "chatxai         0.72\n",
      "chatxaiAuto     0.92\n",
      "chatxaiboost    0.79\n",
      "dashboard       0.89\n",
      "Name: perceived_feature_understanding, dtype: float64\n",
      "-----------------\n",
      "completeness\n",
      "                Source      SS   DF       F  p-unc     n2\n",
      "0            condition   0.471    3   0.383  0.766  0.004\n",
      "1  Propensity to Trust  18.139    1  44.212  0.000  0.155\n",
      "2          Familiarity   0.043    1   0.106  0.746  0.000\n",
      "3                  ATI   0.053    1   0.130  0.719  0.000\n",
      "4         mlbackground   1.151    1   2.805  0.095  0.010\n",
      "5             Residual  97.234  237     NaN    NaN    NaN\n",
      "0.38 & .766 & 44.21 & .000 & 0.11 & .746 & 0.13 & .719 & 2.80 & .095\\\\\n",
      "condition\n",
      "chatxai         3.56\n",
      "chatxaiAuto     3.60\n",
      "chatxaiboost    3.53\n",
      "dashboard       3.60\n",
      "Name: completeness, dtype: float64\n",
      "condition\n",
      "chatxai         0.72\n",
      "chatxaiAuto     0.77\n",
      "chatxaiboost    0.64\n",
      "dashboard       0.67\n",
      "Name: completeness, dtype: float64\n",
      "-----------------\n",
      "coherence\n",
      "                Source       SS   DF       F  p-unc     n2\n",
      "0            condition    3.122    3   1.407  0.241  0.014\n",
      "1  Propensity to Trust   44.698    1  60.451  0.000  0.200\n",
      "2          Familiarity    0.005    1   0.007  0.932  0.000\n",
      "3                  ATI    0.018    1   0.024  0.876  0.000\n",
      "4         mlbackground    0.644    1   0.871  0.352  0.003\n",
      "5             Residual  175.241  237     NaN    NaN    NaN\n",
      "1.41 & .241 & 60.45 & .000 & 0.01 & .932 & 0.02 & .876 & 0.87 & .352\\\\\n",
      "condition\n",
      "chatxai         3.81\n",
      "chatxaiAuto     3.66\n",
      "chatxaiboost    3.87\n",
      "dashboard       3.57\n",
      "Name: coherence, dtype: float64\n",
      "condition\n",
      "chatxai         0.92\n",
      "chatxaiAuto     1.00\n",
      "chatxaiboost    0.99\n",
      "dashboard       0.97\n",
      "Name: coherence, dtype: float64\n",
      "-----------------\n",
      "clarity\n",
      "                Source       SS   DF       F  p-unc     n2\n",
      "0            condition    0.134    3   0.089  0.966  0.001\n",
      "1  Propensity to Trust   34.460    1  68.233  0.000  0.221\n",
      "2          Familiarity    0.806    1   1.596  0.208  0.005\n",
      "3                  ATI    0.504    1   0.997  0.319  0.003\n",
      "4         mlbackground    0.306    1   0.607  0.437  0.002\n",
      "5             Residual  119.694  237     NaN    NaN    NaN\n",
      "0.09 & .966 & 68.23 & .000 & 1.60 & .208 & 1.00 & .319 & 0.61 & .437\\\\\n",
      "condition\n",
      "chatxai         4.02\n",
      "chatxaiAuto     3.98\n",
      "chatxaiboost    4.02\n",
      "dashboard       4.02\n",
      "Name: clarity, dtype: float64\n",
      "condition\n",
      "chatxai         0.76\n",
      "chatxaiAuto     0.83\n",
      "chatxaiboost    0.83\n",
      "dashboard       0.83\n",
      "Name: clarity, dtype: float64\n",
      "-----------------\n",
      "learning_effect\n",
      "                Source       SS   DF       F  p-unc     n2\n",
      "0            condition    2.337    3   1.196  0.312  0.014\n",
      "1  Propensity to Trust   12.122    1  18.608  0.000  0.071\n",
      "2          Familiarity    0.742    1   1.139  0.287  0.004\n",
      "3                  ATI    0.055    1   0.084  0.772  0.000\n",
      "4         mlbackground    0.850    1   1.306  0.254  0.005\n",
      "5             Residual  154.396  237     NaN    NaN    NaN\n",
      "1.20 & .312 & 18.61 & .000 & 1.14 & .287 & 0.08 & .772 & 1.31 & .254\\\\\n",
      "condition\n",
      "chatxai         4.06\n",
      "chatxaiAuto     3.79\n",
      "chatxaiboost    4.03\n",
      "dashboard       3.93\n",
      "Name: learning_effect, dtype: float64\n",
      "condition\n",
      "chatxai         0.77\n",
      "chatxaiAuto     1.00\n",
      "chatxaiboost    0.73\n",
      "dashboard       0.87\n",
      "Name: learning_effect, dtype: float64\n",
      "-----------------\n",
      "understanding_of_system\n",
      "                Source       SS   DF       F  p-unc     n2\n",
      "0            condition    0.911    3   0.459  0.711  0.005\n",
      "1  Propensity to Trust   28.336    1  42.881  0.000  0.150\n",
      "2          Familiarity    0.294    1   0.445  0.505  0.002\n",
      "3                  ATI    2.613    1   3.954  0.048  0.014\n",
      "4         mlbackground    0.001    1   0.001  0.977  0.000\n",
      "5             Residual  156.610  237     NaN    NaN    NaN\n",
      "0.46 & .711 & 42.88 & .000 & 0.44 & .505 & 3.95 & .048 & 0.00 & .977\\\\\n",
      "condition\n",
      "chatxai         3.95\n",
      "chatxaiAuto     3.93\n",
      "chatxaiboost    4.03\n",
      "dashboard       4.13\n",
      "Name: understanding_of_system, dtype: float64\n",
      "condition\n",
      "chatxai         0.86\n",
      "chatxaiAuto     1.03\n",
      "chatxaiboost    0.87\n",
      "dashboard       0.78\n",
      "Name: understanding_of_system, dtype: float64\n",
      "-----------------\n",
      "usefulness_explanation\n",
      "                Source      SS   DF       F  p-unc     n2\n",
      "0            condition   0.618    3   0.545  0.652  0.005\n",
      "1  Propensity to Trust  23.923    1  63.347  0.000  0.209\n",
      "2          Familiarity   0.191    1   0.505  0.478  0.002\n",
      "3                  ATI   0.044    1   0.116  0.733  0.000\n",
      "4         mlbackground   0.025    1   0.067  0.796  0.000\n",
      "5             Residual  89.502  237     NaN    NaN    NaN\n",
      "0.55 & .652 & 63.35 & .000 & 0.50 & .478 & 0.12 & .733 & 0.07 & .796\\\\\n",
      "condition\n",
      "chatxai         4.09\n",
      "chatxaiAuto     3.99\n",
      "chatxaiboost    4.02\n",
      "dashboard       3.95\n",
      "Name: usefulness_explanation, dtype: float64\n",
      "condition\n",
      "chatxai         0.67\n",
      "chatxaiAuto     0.77\n",
      "chatxaiboost    0.59\n",
      "dashboard       0.75\n",
      "Name: usefulness_explanation, dtype: float64\n",
      "-----------------\n",
      "objective_feature_understanding\n",
      "                Source     SS   DF      F  p-unc     n2\n",
      "0            condition  0.053    3  3.219  0.023  0.037\n",
      "1  Propensity to Trust  0.021    1  3.832  0.051  0.015\n",
      "2          Familiarity  0.044    1  8.104  0.005  0.031\n",
      "3                  ATI  0.006    1  1.087  0.298  0.004\n",
      "4         mlbackground  0.000    1  0.043  0.835  0.000\n",
      "5             Residual  1.290  237    NaN    NaN    NaN\n",
      "3.22 & .023 & 3.83 & .051 & 8.10 & .005 & 1.09 & .298 & 0.04 & .835\\\\\n",
      "condition\n",
      "chatxai         0.88\n",
      "chatxaiAuto     0.84\n",
      "chatxaiboost    0.87\n",
      "dashboard       0.88\n",
      "Name: objective_feature_understanding, dtype: float64\n",
      "condition\n",
      "chatxai         0.08\n",
      "chatxaiAuto     0.06\n",
      "chatxaiboost    0.09\n",
      "dashboard       0.07\n",
      "Name: objective_feature_understanding, dtype: float64\n",
      "-----------------\n"
     ]
    }
   ],
   "source": [
    "print(\"-\" * 34)\n",
    "from pingouin import ancova, anova\n",
    "print(\"For all participants, compare with experimental conditions\")\n",
    "for dimension in dimensions:\n",
    "    print(dimension)\n",
    "    # aov = anova(dv=dimension, between='condition', data=df, effsize='n2')\n",
    "    aov = ancova(dv=dimension, covar=[\"Propensity to Trust\", \"Familiarity\", \"ATI\", \"mlbackground\"], between='condition', data=df, effsize='n2')\n",
    "    print(aov.round(3))\n",
    "    tp_dict = aov.to_dict()\n",
    "    # f_list = [\"{:.2f}\".format(tp_dict['F'][index]) for index in range(0, 5)]\n",
    "    # p_list = [\"{:.2f}\".format(tp_dict['p-unc'][index]) for index in range(0, 5)]\n",
    "    tp_str = \"{:.2f} & {}\".format(tp_dict['F'][0], \"{:.3f}\".format(tp_dict['p-unc'][0])[1:])\n",
    "    for index in range(1,5):\n",
    "        tp_str += \" & {:.2f} & {}\".format(tp_dict['F'][index], \"{:.3f}\".format(tp_dict['p-unc'][index])[1:])\n",
    "    tp_str += \"\\\\\\\\\"\n",
    "    print(tp_str)\n",
    "    print(df.groupby(['condition'])[dimension].mean().round(2))\n",
    "    print(df.groupby(['condition'])[dimension].std().round(2))\n",
    "    print(\"-\" * 17)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "objective_feature_understanding\n",
      "kruskal test result: H:16.19, p:0.001\n",
      "61, Mean: M(dashboard):0.88, SD(dashboard):0.07\n",
      "62, Mean: M(chatxai):0.88, SD(chatxai):0.08\n",
      "61, Mean: M(chatxaiboost):0.87, SD(chatxaiboost):0.08\n",
      "61, Mean: M(chatxaiAuto):0.84, SD(chatxaiAuto):0.06\n",
      "----------------------------------\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "Alternative dashboard > chatxaiAuto, pvalue 0.0007 statistic 2487.5000\n",
      "Use pots-hoc analysis\n",
      "No significant difference with post-hoc analysis\n",
      "Use pots-hoc analysis\n",
      "Alternative chatxai > chatxaiAuto, pvalue 0.0004 statistic 2559.0000\n",
      "Use pots-hoc analysis\n",
      "Alternative chatxaiboost > chatxaiAuto, pvalue 0.0006 statistic 2492.5000\n",
      "----------------------------------\n"
     ]
    }
   ],
   "source": [
    "for dimension in [\"objective_feature_understanding\"]:\n",
    "    print(dimension)\n",
    "    var_dict = {}\n",
    "    all_conditions = [\"dashboard\", \"chatxai\", \"chatxaiboost\", \"chatxaiAuto\"]\n",
    "    for condition in all_conditions:\n",
    "        tp_df = df[df['condition'] == condition]\n",
    "        var_dict[condition] = tp_df[dimension].tolist()\n",
    "        # print(len(var_dict[condition]))\n",
    "    kwargs = [var_dict[condition] for condition in all_conditions]\n",
    "    statistic, pvalue = kruskal(*kwargs)\n",
    "    print(\"kruskal test result: H:{:.2f}, p:{:.3f}\".format(statistic, pvalue))\n",
    "    for condition in all_conditions:\n",
    "        data_list_1 = var_dict[condition]\n",
    "        print(\"{}, Mean: M({}):{:.2f}, SD({}):{:.2f}\".format(len(data_list_1), condition, np.mean(data_list_1), condition, np.std(data_list_1)))\n",
    "    if pvalue < 0.05 / 4:\n",
    "        length = len(all_conditions)\n",
    "        print(\"-\" * 34)\n",
    "        for i in range(length - 1):\n",
    "            for j in range(i+1, length):\n",
    "                group_1 = all_conditions[i]\n",
    "                group_2 = all_conditions[j]\n",
    "                data_list_1 = var_dict[group_1]\n",
    "                data_list_2 = var_dict[group_2]\n",
    "                post_hoc_comparison(data_list_1, data_list_2, group_1, group_2)\n",
    "        print(\"-\" * 34)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "agreement_phi",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
