Lesson-02/Search Problem.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. Get data from web page."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import re\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\urllib3\\connectionpool.py:847: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    }
   ],
   "source": [
    "#参考文章：https://blog.csdn.net/HollyRan/article/details/85254375\n",
    "#目标网站：https://www.bjsubway.com/station/zjgls/#\n",
    "#百度百科的站点不太靠谱，从北京地铁官网找到的这个比较好，还包括站点距离\n",
    "\n",
    "url = 'https://www.bjsubway.com/station/zjgls/#'\n",
    "#verify必须等于false，否则SSL会报错\n",
    "response = requests.get(url,verify=False)\n",
    "#使用中文编码格式gbk进行解码\n",
    "response.encoding = 'gbk'\n",
    "html = response.text\n",
    "#将源代码信息用BeautifulSoup的parser解码器进行解码\n",
    "soup = BeautifulSoup(html,\"html.parser\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<td colspan=\"5\">1号线相邻站间距信息统计表</td>, <td colspan=\"5\">5号线相邻站间距信息统计表</td>, <td colspan=\"5\">6号线相邻站间距信息统计表</td>, <td colspan=\"5\">7号线相邻站间距信息统计表</td>, <td colspan=\"5\">8号线相邻站间距信息统计表</td>, <td colspan=\"5\">9号线相邻站间距信息统计表</td>, <td colspan=\"5\">八通线相邻站间距信息统计表</td>, <td colspan=\"5\">亦庄线相邻站间距信息统计表</td>, <td colspan=\"5\">房山线相邻站间距信息统计表</td>, <td colspan=\"5\">机场线相邻站间距信息统计表</td>, <td colspan=\"6\">15号线相邻站间距信息统计表</td>, <td colspan=\"6\">昌平线相邻站间距信息统计表</td>, <td colspan=\"7\">2号线相邻站间距信息统计表</td>, <td colspan=\"7\">4号线相邻站间距信息统计表</td>, <td colspan=\"7\">13号线相邻站间距信息统计表</td>, <td colspan=\"7\">14号线(西段)相邻站间距信息统计表</td>, <td colspan=\"7\">14号线（东段）相邻站间距信息统计表</td>, <td colspan=\"7\">大兴线相邻站间距信息统计表</td>, <td colspan=\"9\">10号线相邻站间距信息统计表</td>]\n",
      "\n",
      "\n",
      "['1号线相邻站间距信息统计表', '5号线相邻站间距信息统计表', '6号线相邻站间距信息统计表', '7号线相邻站间距信息统计表', '8号线相邻站间距信息统计表', '9号线相邻站间距信息统计表', '八通线相邻站间距信息统计表', '亦庄线相邻站间距信息统计表', '房山线相邻站间距信息统计表', '机场线相邻站间距信息统计表', '15号线相邻站间距信息统计表', '昌平线相邻站间距信息统计表', '2号线相邻站间距信息统计表', '4号线相邻站间距信息统计表', '13号线相邻站间距信息统计表', '14号线(西段)相邻站间距信息统计表', '14号线（东段）相邻站间距信息统计表', '大兴线相邻站间距信息统计表', '10号线相邻站间距信息统计表']\n"
     ]
    }
   ],
   "source": [
    "# 得到线路名称\n",
    "def get_txt_name():  \n",
    "    txt_src_name = []\n",
    "    for i in range(5, 10):\n",
    "        #观察源代码可知线路信息都在名为\"colspan\"的字符串后面，且colspan后面的数字在5-10之间\n",
    "        #使用soup自带的正则表达，字典方式\n",
    "        temp = soup.find_all('td', {'colspan': str(i)})\n",
    "        txt_src_name += temp\n",
    "    return txt_src_name\n",
    "print(get_txt_name())\n",
    "#得到准确线路名称，去除边角冗余字符串\n",
    "def get_txtuseful_name(): \n",
    "    obj = []\n",
    "    for each in get_txt_name():\n",
    "        # 从>匹配到<(不包含)，用括号包住想要的部分\n",
    "        temp = re.findall(r\">(.+?)<\", str(each))  \n",
    "        obj += temp\n",
    "    return obj\n",
    "print('\\n')\n",
    "#查看地铁线路总共有多少\n",
    "print(get_txtuseful_name())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#分析网页源码，发现所有的站点信息都被存在\"<tbody>\"下\n",
    "Station_info = soup.find_all('tbody')\n",
    "#print(Station_info)\n",
    "#tbody下的关键信息仍然需要正则提取\n",
    "def get_station_info():\n",
    "    obj = []\n",
    "    for each in Station_info:\n",
    "        temp = re.findall(r\">(.+?)<\", str(each))   # 正则匹配，str格式\n",
    "        obj += temp\n",
    "    return obj\n",
    "#print(get_station_info())\n",
    "#得到站点与位置两个关键信息\n",
    "station_geo_info = get_station_info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. Preprocessing data from page source."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#保存到test文件备用\n",
    "with open('test.txt','w',encoding='utf-8') as f:\n",
    "    for line in station_geo_info:\n",
    "        if line == '上行/下行' or line == '上行' or line == '下行':\n",
    "            f.write('\\n')\n",
    "        else:\n",
    "            f.write(line + '   ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#保存到列表中\n",
    "info = []\n",
    "for i in station_geo_info:\n",
    "    #去除多余的文字\n",
    "    if i == '上行/下行' or i == '上行' or i == '下行': continue\n",
    "    else:\n",
    "        info.append(i)\n",
    "        \n",
    "info_1 = {}\n",
    "n = 0\n",
    "for i in info:\n",
    "    n += 1\n",
    "    #站点的字符串大于5，选出站点形成字典\n",
    "    #站点作为key，距离作为值，把站点间的字符串‘--’换成‘:’\n",
    "    if len(i)>5:\n",
    "        info_1[i.replace('——',':')] = info[n]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#先找出站点间单向连接的关系\n",
    "station_connection_single = {}\n",
    "visited = []\n",
    "for i in info_1.keys():\n",
    "    #如果某站点在之前出现过，那么此站点还能到达其他点\n",
    "    #如建国门可到北京站，也可到东单，先设{'建国门':['北京站']},再把东单append进入key对应的value中\n",
    "    if i.split(':')[0] in visited:\n",
    "        station_connection_single[i.split(':')[0]].append(i.split(':')[1])\n",
    "    else:\n",
    "        station_connection_single[i.split(':')[0]]= [i.split(':')[1]]\n",
    "        visited.append(i.split(':')[0])\n",
    "#station_connection_single"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#补充前后连接关系\n",
    "#一个站点除了可以到它的下一个点，它还可以去到它的上一个点\n",
    "station_connection = station_connection_single\n",
    "all_station = list(station_connection_single.keys())\n",
    "c = list(station_connection_single.values())\n",
    "for i in all_station:\n",
    "    for p in c:\n",
    "        #若此站点出现在其他站点可以到达的站点中，则此站点也能到达那个站点\n",
    "        #如建国门可到站点为[北京站，东单],对于北京站和东单而言，它们也可以到建国门\n",
    "        #找到建国门的索引号，赋入字典key对应的value中\n",
    "        if i in p:\n",
    "            index = c.index(p)\n",
    "            station_connection[i].append(all_station[index])        \n",
    "#station_connection     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['天安门西', '宣武门', '复兴门', '灵境胡同']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "station_connection['西单'] "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3. Build the search agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#和课堂代码一样，创建搜索策略\n",
    "def search(start,stop,connection_graph,sort_candidate):\n",
    "    pathes = [[start]]\n",
    "    visited = set()\n",
    "    \n",
    "    while pathes:\n",
    "        path = pathes.pop()\n",
    "        frontier = path[-1]\n",
    "        if frontier in visited: continue\n",
    "        successors = connection_graph[frontier]\n",
    "        for city in successors:\n",
    "            if city in path: continue\n",
    "            new_path = path + [city]\n",
    "            pathes.append(new_path)\n",
    "            if city == stop: return new_path\n",
    "        visited.add(frontier)\n",
    "        pathes = sort_candidate(pathes)\n",
    "#最长站点\n",
    "def transfer_as_much(pathes):\n",
    "    return sorted(pathes,key=len)\n",
    "#最少站点\n",
    "def transfer_as_less(pathes):\n",
    "    return sorted(pathes,key=len,reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#导入emoji\n",
    "from emoji import emojize\n",
    "def pretty_print(cities):\n",
    "    print(emojize(\" :metro:-> \").join(cities))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "建国门 🚇-> 东单 🚇-> 王府井 🚇-> 天安门东 🚇-> 天安门西 🚇-> 西单 🚇-> 复兴门\n"
     ]
    }
   ],
   "source": [
    "pretty_print(search('建国门','复兴门',station_connection,sort_candidate=transfer_as_less))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "建国门 🚇-> 北京站 🚇-> 崇文门 🚇-> 前门 🚇-> 和平门 🚇-> 宣武门 🚇-> 长椿街 🚇-> 复兴门\n"
     ]
    }
   ],
   "source": [
    "pretty_print(search('建国门','复兴门',station_connection,sort_candidate=transfer_as_much))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "东单 🚇-> 灯市口 🚇-> 东四 🚇-> 南锣鼓巷 🚇-> 什刹海 🚇-> 鼓楼大街 🚇-> 安德里北街 🚇-> 安华桥 🚇-> 北土城 🚇-> 奥体中心 🚇-> 奥林匹克公园 🚇-> 北沙滩 🚇-> 六道口\n"
     ]
    }
   ],
   "source": [
    "pretty_print(search('东单','六道口',station_connection,sort_candidate=transfer_as_less))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}