{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "pycharm": {
     "is_executing": true
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from datasets import load_dataset\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "is_executing": true
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#fetch devin's evaluation outputs into a dataframe\n",
    "def get_devin_eval_output():\n",
    "    repo_url = \"CognitionAI/devin-swebench-results\"\n",
    "    folder_path = \"output_diffs\"\n",
    "\n",
    "    base_url = \"https://api.github.com/repos/\"\n",
    "    pass_api_url = f\"{base_url}{repo_url}/contents/{folder_path}/pass\"\n",
    "    failed_api_url = f\"{base_url}{repo_url}/contents/{folder_path}/fail\"\n",
    "\n",
    "    files_info = []\n",
    "\n",
    "    def get_files(api_url, subfolder_name):\n",
    "        response = requests.get(api_url)\n",
    "        if response.status_code == 200:\n",
    "            contents = response.json()\n",
    "            for item in tqdm(contents):\n",
    "                if item[\"type\"] == \"file\":\n",
    "                    file_url = f\"https://raw.githubusercontent.com/{repo_url}/main/{folder_path}/{subfolder_name}/{item['name']}\"\n",
    "                    file_content = requests.get(file_url).text\n",
    "                    files_info.append({\"instance_id\":item['name'][:-9],\"content\": file_content, \"pass or fail\": subfolder_name})\n",
    "\n",
    "    get_files(pass_api_url, \"pass\")\n",
    "    get_files(failed_api_url, \"fail\")\n",
    "\n",
    "    df = pd.DataFrame(files_info)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get the swe test dataset\n",
    "swe_df = pd.DataFrame(load_dataset(\"princeton-nlp/SWE-bench\", split=\"test\"))\n",
    "id2repo = {i:repo for i,repo in zip(swe_df.instance_id, swe_df.repo)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = get_devin_eval_output()\n",
    "df['repo'] = [id2repo[i] for i in df.instance_id]\n",
    "df.to_csv('../data/devin_output_diffs.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, \"Devin's pass/fail status across repos\")"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 54080 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# check the pass/fail status of devin outputs across different repos\n",
    "sns.countplot(y='repo', hue='pass or fail', data=df)\n",
    "plt.title('Devin\\'s pass/fail status across repos')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, '%Resolved')"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 51212 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "repo_counts = df.groupby(['repo', 'pass or fail']).size().unstack(fill_value=0)\n",
    "repo_counts['ratio'] = repo_counts['pass'] / (repo_counts['pass'] + repo_counts['fail'])\n",
    "sns.barplot(y=repo_counts.index, x='ratio', data=repo_counts.reset_index(), order = list(df.repo.drop_duplicates()))\n",
    "plt.title('Devin\\'s issue-resolve rate across repos')\n",
    "plt.xlabel('%Resolved')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Number of Test Instances Under Each Repo: Devin vs. SWE-bench')"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 60364 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#check the distribution differences between SWE-bench test set and the subset devin used\n",
    "swe_df['subset'] = 'SWE-bench_test'\n",
    "df['subset'] = 'devin'\n",
    "merged_df = pd.concat([df[['subset', 'repo']], swe_df[['subset', 'repo']]])\n",
    "sns.countplot(y='repo', hue='subset', data=merged_df)\n",
    "plt.title('Number of Test Instances Under Each Repo: Devin vs. SWE-bench')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, '')"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 49708 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "repo_counts = merged_df.groupby(['repo', 'subset']).size().unstack(fill_value=0)\n",
    "repo_counts['ratio'] = repo_counts['devin'] / (repo_counts['devin'] + repo_counts['SWE-bench_test'])\n",
    "sns.barplot(y=repo_counts.index, x='ratio', data=repo_counts.reset_index(), order = list(df.repo.drop_duplicates()))\n",
    "plt.title('Repo-level sampling rate of Devin\\'s test set')\n",
    "plt.xlabel('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
