automated-issue-ref.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f64573a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# automated ref adding for lineage-note.txt, written by Xu Zou\n",
    "import bs4\n",
    "from bs4 import BeautifulSoup\n",
    "from urllib import request\n",
    "\n",
    "# the number of milestone we start to read\n",
    "start_milestone=1\n",
    "start_sub_milestone=1\n",
    "def read_milestone(num,di='cov-lineages/pango-designation'):\n",
    "    # or sars-cov-2 proj if it starts to add milestones\n",
    "    url=\"https://github.com/\"+di+\"/milestone/\"+str(num)\n",
    "    res=request.urlopen(url)\n",
    "    rp=res.read()\n",
    "    rp=rp.decode('utf-8')\n",
    "    html_soup=BeautifulSoup(rp,\"html\")\n",
    "    title=html_soup.title\n",
    "    progress_percent=html_soup.findAll(class_=\"progress-percent\")[0].text\n",
    "    #updated version, ignore those with 0% progress.\n",
    "    if progress_percent==\"0%\":\n",
    "        return []\n",
    "    \n",
    "    title=title.text.replace(',','')\n",
    "   \n",
    "    title=title.split()[:-3]\n",
    "    \n",
    "    return title\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "769f2d32",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "796 HTTP Error 404: Not Found\n",
      "8 HTTP Error 404: Not Found\n"
     ]
    }
   ],
   "source": [
    "# read milestones \n",
    "milestone_end=False\n",
    "milestone=start_milestone\n",
    "milestone_list=[]\n",
    "milestone_list_sub=[]\n",
    "milestone_sub=start_sub_milestone\n",
    "import os \n",
    "listd=os.listdir()\n",
    "\n",
    "if not(\"autoref\" in listd):\n",
    "    os.mkdir(\"autoref\")\n",
    "lista=os.listdir(\"autoref\")\n",
    "# reading milestones is very slow, so we save already read milestones to autoref/milestones.txt\n",
    "if (\"milestone.txt\" in lista):\n",
    "    w=open(\"autoref/milestone.txt\",'r')\n",
    "    milestones=w.readlines()\n",
    "    for line in milestones:\n",
    "        linsp=line.split()\n",
    "        if len(linsp)>=2:\n",
    "            \n",
    "            issue=int(linsp[0])\n",
    "            name=linsp[1]\n",
    "            if (linsp[1]!='·' and linsp[1]!='Milestone'):\n",
    "                milestone_list.append([linsp[0],linsp[1]])\n",
    "            # don't read twice for old milestones\n",
    "            if issue>=milestone:\n",
    "                milestone=issue+1          \n",
    "    w.close()\n",
    "\n",
    "if (\"milestone_sub.txt\" in lista):\n",
    "    w=open(\"autoref/milestone_sub.txt\",'r')\n",
    "    milestones=w.readlines()\n",
    "    for line in milestones:\n",
    "        linsp=line.split()\n",
    "        if len(linsp)>=2:\n",
    "            issue=int(linsp[0])\n",
    "            name=linsp[1]\n",
    "            if (linsp[1]!='·' and linsp[1]!='Milestone'):\n",
    "                milestone_list_sub.append([linsp[0],linsp[1]])\n",
    "            # don't read twice for old milestones\n",
    "            if issue>=milestone_sub:\n",
    "                milestone_sub=issue+1          \n",
    "    w.close()  \n",
    "#rewrite the txt\n",
    "\n",
    "w=open(\"autoref/milestone.txt\",'w',newline='\\n')\n",
    "for item in milestone_list:\n",
    "    print(item[0],item[1],file=w)\n",
    "\n",
    "\n",
    "while not(milestone_end):\n",
    "    try:\n",
    "        title=read_milestone(milestone)\n",
    "        for item in title:\n",
    "            milestone_list.append([milestone,item])\n",
    "            print(milestone,item,file=w)\n",
    "    except Exception as e:\n",
    "        print(milestone,e)\n",
    "        milestone_end=True\n",
    "    milestone+=1\n",
    "    if milestone%10==0:\n",
    "        print(\"Reading Milestone\",milestone)\n",
    "\n",
    "w.close()\n",
    "\n",
    "w=open(\"autoref/milestone_sub.txt\",'w',newline='\\n')\n",
    "for item in milestone_list_sub:\n",
    "    print(item[0],item[1],file=w)\n",
    "\n",
    "milestone_end=False\n",
    "while not(milestone_end):\n",
    "    try:\n",
    "        title=read_milestone(milestone_sub,di=\"sars-cov-2-variants/lineage-proposals\")\n",
    "        for item in title:\n",
    "            milestone_list_sub.append([milestone_sub,item])\n",
    "            print(milestone_sub,item,file=w)\n",
    "    except Exception as e:\n",
    "        print(milestone_sub,e)\n",
    "        milestone_end=True\n",
    "    milestone_sub+=1\n",
    "    if milestone%10==0:\n",
    "        print(\"Reading Milestone in sars-cov-2-variants \",milestone)\n",
    "\n",
    "w.close()\n",
    "\n",
    "    \n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "88a367db",
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_milestone_issues(name,di='cov-lineages/pango-designation'):\n",
    "    # or sars-cov-2 proj if it starts to add milestones\n",
    "    \n",
    "    # first, deal with multiple names\n",
    "    to_add=all_milestones[name]\n",
    "    actual_milestone='\"'+item_to_milestone[to_add]+'\"'\n",
    "    url=\"https://github.com/\"+di+\"/issues?q=milestone:\"+actual_milestone\n",
    "    res=request.urlopen(url)\n",
    "    rp=res.read()\n",
    "    rp=rp.decode('utf-8')\n",
    "    html_soup=BeautifulSoup(rp,\"html\")\n",
    "    html=html_soup.find_all(class_=\"Link--primary v-align-middle no-underline h4 js-navigation-open markdown-title\" )\n",
    "    try:\n",
    "        id_str=html[0]['id']\n",
    "        outid=id_str.split(\"_\")[1]\n",
    "        return int(outid)\n",
    "    except Exception as e:\n",
    "        print(name,e)\n",
    "        return 0\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a56544a8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['BA.2.75.9', '#1098']\n",
      "['CB.1', '#1098']\n",
      "['CK.2', '#1190']\n",
      "['CK.2.1.1', '#1190']\n",
      "['BA.5.2.36', '#1203']\n",
      "['CT.1', '#1203']\n",
      "['BA.5.2.38', '#1254']\n",
      "['DA.1', '#1254']\n",
      "['AY.13', '#198']\n",
      "['AY.14', '#198']\n",
      "['AY.15', '#198']\n",
      "['AY.16', '#198']\n",
      "['AY.17', '#198']\n",
      "['AY.18', '#198']\n",
      "['AY.19', '#198']\n",
      "['AY.20', '#198']\n",
      "['AY.21', '#198']\n",
      "['AY.22', '#198']\n",
      "['AY.34.1.1', '#255']\n",
      "['AY.58', '#286']\n",
      "['AY.63', '#286']\n",
      "['AY.99', '#258']\n",
      "['AY.99.1', '#258']\n",
      "['AY.99.2', '#258']\n",
      "['AY.112.1', '#283']\n",
      "['AY.121.1', '#344']\n",
      "['AY.122', '#320']\n",
      "['EG.4.2', 'sars-cov-2-variants/lineage-proposals/#46']\n",
      "['GY.6', 'sars-cov-2-variants/lineage-proposals/#308']\n",
      "['XBB.1.32', '#1827']\n",
      "['XBB.1.37', '#1929']\n",
      "['XBF.1.1', '#1651']\n"
     ]
    }
   ],
   "source": [
    "# read lineage_notes.txt\n",
    "\n",
    "def milestone2issue(num):\n",
    "    return 0\n",
    "w=open(\"lineage_notes.txt\",'r')\n",
    "lineage_notes=w.readlines()\n",
    "w.close()\n",
    "milestone_tobecheck=[]\n",
    "#create a name-milestone match for milestones\n",
    "all_milestones={}\n",
    "item_to_milestone={}\n",
    "for item in milestone_list:\n",
    "    to_add=\"#\"+str(item[0])\n",
    "    all_milestones[item[1]]=to_add\n",
    "    if to_add in item_to_milestone:\n",
    "        item_to_milestone[to_add]+='+'+item[1]\n",
    "    else:\n",
    "        item_to_milestone[to_add]=item[1]\n",
    "        \n",
    "for item in milestone_list_sub:\n",
    "    to_add='sars-cov-2-variants/lineage-proposals/#'+str(item[0])\n",
    "    all_milestones[item[1]]=to_add\n",
    "    if to_add in item_to_milestone:\n",
    "        item_to_milestone[to_add]+='+'+item[1]\n",
    "    else:\n",
    "        item_to_milestone[to_add]=item[1]\n",
    "\n",
    "changed_items=0\n",
    "missing_milestones=[]\n",
    "for num in range(1,len(lineage_notes)):\n",
    "    line=lineage_notes[num]\n",
    "    linsp=line.split()\n",
    "    if (len(linsp)>=2):\n",
    "        title=linsp[0]\n",
    "        # for all lineages without an issue label but with a milestone\n",
    "        if not(\"#\" in line):\n",
    "            if title in all_milestones:\n",
    "                # this time we need to deal with the milestone \n",
    "                if 'sars' in all_milestones[title]:\n",
    "                    issue_num=read_milestone_issues(title,di='sars-cov-2-variants/lineage-proposals')\n",
    "                    if issue_num!=0:\n",
    "                        missing_milestones.append([title,'sars-cov-2-variants/lineage-proposals/#'+str(issue_num)])\n",
    "                        print(missing_milestones[-1])\n",
    "                else:\n",
    "                    issue_num=read_milestone_issues(title)\n",
    "                    if issue_num!=0:\n",
    "                        missing_milestones.append([title,'#'+str(issue_num)])\n",
    "                        print(missing_milestones[-1])\n",
    "                    \n",
    "#rewrite lineage_notes.txt\n",
    "w=open(\"autoref/missing_refs.txt\",'w')\n",
    "for item in missing_milestones:\n",
    "    print(item[0],item[1],file=w)\n",
    "w.close()\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1c9369e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#final rewrite of lineage_notes.txt\n",
    "w=open(\"autoref/missing_refs.txt\",'r')\n",
    "data=w.readlines()\n",
    "w.close()\n",
    "missing_refs={}\n",
    "for line in data:\n",
    "    linsp=line.split()\n",
    "    missing_refs[linsp[0]]=linsp[1].replace('\\n','')\n",
    "\n",
    "w=open(\"lineage_notes.txt\",'r')\n",
    "data=w.readlines()\n",
    "w.close()\n",
    "for i in range(len(data)):\n",
    "    linsp=data[i].split()\n",
    "    if linsp[0] in missing_refs:\n",
    "        data[i]=data[i].replace('\\n','')+\" From \"+missing_refs[linsp[0]]+'\\n'\n",
    "\n",
    "w=open(\"lineage_notes.txt\",'w',newline='\\n')\n",
    "w.writelines(data)\n",
    "w.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7756372f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b3798f0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}