index.html

<!--      　　　___           ___           ___           ___           ___                    ___           ___       ___           ___     
　　　　　　   /\  \         /\  \         /\__\         |\__\         /\  \                  /\  \         /\__\     /\  \         /\  \    
  　　　　　　 \:\  \       /::\  \       /:/  /         |:|  |       /::\  \                /::\  \       /:/  /    /::\  \       /::\  \   
          　　  \:\  \     /:/\:\  \     /:/__/          |:|  |      /:/\ \  \              /:/\:\  \     /:/  /    /:/\:\  \     /:/\:\  \  
          　　  /::\  \   /::\~\:\  \   /::\  \ ___      |:|__|__   _\:\~\ \  \            /::\~\:\__\   /:/  /    /:/  \:\  \   /:/  \:\  \ 
          　　 /:/\:\__\ /:/\:\ \:\__\ /:/\:\  /\__\ ____/::::\__\ /\ \:\ \ \__\          /:/\:\ \:|__| /:/__/    /:/__/ \:\__\ /:/__/_\:\__\
        　　  /:/  \/__/ \/_|::\/:/  / \/__\:\/:/  / \::::/~~/~    \:\ \:\ \/__/          \:\~\:\/:/  / \:\  \    \:\  \ /:/  / \:\  /\ \/__/
        　　 /:/  /         |:|::/  /       \::/  /   ~~|:|~~|      \:\ \:\__\             \:\ \::/  /   \:\  \    \:\  /:/  /   \:\ \:\__\  
        　　 \/__/          |:|\/__/        /:/  /      |:|  |       \:\/:/  /              \:\/:/  /     \:\  \    \:\/:/  /     \:\/:/  /  
        　　                |:|  |         /:/  /       |:|  |        \::/  /                \::/__/       \:\__\    \::/  /       \::/  /   
        　　                 \|__|         \/__/         \|__|         \/__/                  ~~            \/__/     \/__/         \/__/    

   ┌───┐   ┌───┬───┬───┬───┐   ┌───┬───┬───┬───┐   ┌───┬───┬───┬───┐  ┌───┬───┬───┐
   │ Esc  │   │  F1  │  F2  │  F3  │  F4  │   │  F5  │  F6  │  F7  │  F8  │   │  F9  │ F10  │ F11  │ F12  │  │ P/S  │ S L  │ P/B  │
   └───┘   └───┴───┴───┴───┘   └───┴───┴───┴───┘   └───┴───┴───┴───┘  └───┴───┴───┘
   ┌───┬───┬───┬───┬───┬───┬───┬───┬───┬───┬───┬───┬───┬───────┐ ┌───┬───┬───┐
   │ ~ `  │ ! 1  │ @ 2  │ # 3  │ $ 4  │ % 5  │ ^ 6  │ & 7  │ * 8  │ ( 9  │ ) 0  │ _ -  │ + =  │    BacSp     │ │ Ins  │ Hom  │ PUp  │
   ├───┴─┬─┴─┬─┴─┬─┴─┬─┴─┬─┴─┬─┴─┬─┴─┬─┴─┬─┴─┬─┴─┬─┴─┬─┴─┬─────┤ ├───┼───┼───┤
   │   Tab    │  Q   │  W   │  E   │  R   │  T   │  Y   │  U   │  I   │  O   │  P   │ { [  │ } ]  │   | \    │ │ Del  │ End  │ PDn  │
   ├─────┴┬──┴┬──┴┬──┴┬──┴┬──┴┬──┴┬──┴┬──┴┬──┴┬──┴┬──┴┬──┴─────┤ └───┴───┴───┘
   │    Caps    │  A   │  S   │  D   │  F   │  G   │  H   │  J   │  K   │  L   │ : ;  │ " '  │     Enter      │
   ├──────┴─┬─┴─┬─┴─┬─┴─┬─┴─┬─┴─┬─┴─┬─┴─┬─┴─┬─┴─┬─┴─┬─┴────────┤         ┌───┐
   │     Shift      │  Z   │  X   │  C   │  V   │  B   │  N   │  M   │ < ,  │ > .  │ ? /  │       Shift        │         │  ↑  │
   ├─────┬──┴─┬─┴──┬┴───┴───┴───┴───┴───┴──┬┴───┼───┴┬────┬────┤ ┌───┼───┼───┐
   │   Ctrl   │        │  Alt   │                     Space                    │   Alt  │   Fn   │   ＝   │  Ctrl  │ │  ←  │  ↓  │  →  │
   └─────┴────┴────┴───────────────────────┴────┴────┴────┴────┘ └───┴───┴───┘

              写字楼里写字间，写字间里程序员；程序人员写程序，又拿程序换酒钱。酒醒只在网上坐，酒醉还来网下眠；酒醉酒醒日复日，网上网下年复年。
              但愿老死电脑间，不愿鞠躬老板前；奔驰宝马贵者趣，公交自行程序员。别人笑我忒疯癫，我笑自己命太贱；不见满街漂亮妹，哪个归得程序员？
-->
<!DOCTYPE html>
<html>
<head>
  <meta charset="utf-8">
  <meta http-equiv="X-UA-Compatible" content="IE=edge" />
  
  <title>Python3 爬虫学习笔记 C01 | TRHX&#39;S BLOG</title>
  
  <meta name="keywords" content="Python爬虫,数据分析,数据可视化,web前端,Java,Python,HTML,软件工程,大学生,hexo,GitHubPages,CodingPages">
  
  
  <meta name="description" content="TRHX 的个人博客；主攻 Python、爬虫、WEB前端、大数据、数据分析、数据可视化；求知若饥，虚心若愚，一入 IT 深似海，从此学习无绝期，记录毕生所学！">
  

  <link rel="alternate" href="/atom.xml" title="TRHX'S BLOG">
  

  <meta name="HandheldFriendly" content="True" />
  <meta name="apple-mobile-web-app-capable" content="yes">
  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
  <!-- 百度广告联盟 -->
  <meta name="baidu_union_verify" content="9fb8c8d4f8c021b77200011d2bcf0c73">
  <!-- Google Adsense -->
  <!--<script data-ad-client="ca-pub-1913211097936916" async src="https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js"></script>-->
  <!-- meta -->
  
  
  <meta name='theme-color' content='#f24e32'>
  <meta name='msapplication-TileColor' content='#f24e32'>
  <meta name='msapplication-config' content='https://cdn.jsdelivr.net/gh/xaoxuu/assets@master/favicon/favicons/browserconfig.xml'>
  
  <!-- 360 -->
  <meta name="360-site-verification" content="d98acd5dccc92e495179410dd58186be" />
  <!-- sogou -->
  <meta name="sogou_site_verification" content="l0yGeAV56W"/>
  <!-- link -->
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/gh/fancyapps/fancybox@3.5.7/dist/jquery.fancybox.min.css" />
  
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/node-waves@0.7.6/dist/waves.min.css">
  
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@5.6.3/css/all.min.css">
  
  
  <link rel='shortcut icon' type='image/x-icon' href='https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.4/images/favicon.ico'>
  <link rel='icon' type='image/x-icon' sizes='32x32' href='https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.4/images/favicon-32x32.png'>
  <link rel='apple-touch-icon' type='image/png' sizes='180x180' href='https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.4/images/apple-touch-icon.png'>
  <link rel='mask-icon' color='#f24e32' href='https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.4/images/safari-pinned-tab.svg'>
  <link rel='manifest' href='https://cdn.jsdelivr.net/gh/xaoxuu/cdn-favicon@19.9.7/site.webmanifest'>
  

  <link rel="shortcut icon" type='image/x-icon' href="https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.4/images/favicon.ico">
  

    <link rel="stylesheet" href="https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.7/css/style.css">
  

  <script>
    function setLoadingBarProgress(num) {
      document.getElementById('loading-bar').style.width=num+"%";
    }
  </script>

  
    <!-- ba -->
    <script>
    var _hmt = _hmt || [];
    (function() {
      var hm = document.createElement("script");
      hm.src = "https://hm.baidu.com/hm.js?df0bc7c6bdbd80356ba4db429724ccad";
      var s = document.getElementsByTagName("script")[0];
      s.parentNode.insertBefore(hm, s);
    })();
    </script>
  
</script>
</head>

<body>
  
  
<div class="cover-wrapper">
  <cover class='cover post half'>
    
    
  <img class='logo' src='https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.6/images/logo.png'/>


  <div class="m_search">
    <form name="searchform" class="form u-search-form">
      <input type="text" class="input u-search-input" placeholder="世界之大，探索一下！" />
      <i class="icon fas fa-search fa-fw"></i>
    </form>
  </div>

<div class='menu navgation'>
  <ul class='h-list'>
    
      
        <li>
          <a class="nav home" href="/"
            
            
            id="home">
            <i class='fas fa-home fa-fw'></i>&nbsp;主页
          </a>
        </li>
      
        <li>
          <a class="nav home" href="/categories/"
            
            
            id="categories">
            <i class='fas fa-folder-open fa-fw'></i>&nbsp;分类
          </a>
        </li>
      
        <li>
          <a class="nav home" href="/tags/"
            
            
            id="tags">
            <i class='fas fa-tags fa-fw'></i>&nbsp;标签
          </a>
        </li>
      
        <li>
          <a class="nav home" href="/friends/"
            
            
            id="friends">
            <i class='fas fa-users fa-fw'></i>&nbsp;朋友
          </a>
        </li>
      
        <li>
          <a class="nav home" href="/comments/"
            
            
            id="comments">
            <i class='fas fa-comments fa-fw'></i>&nbsp;留言
          </a>
        </li>
      
    
  </ul>
</div>

    
    <br>
    <!-- Hitokoto 一言 -->
    <!--<p id="hitokoto"></p>
			<script src="https://cdn.jsdelivr.net/npm/bluebird@3/js/browser/bluebird.min.js"></script>
			<script src="https://cdn.jsdelivr.net/npm/whatwg-fetch@2.0.3/fetch.min.js"></script>
      <script src="https://v1.hitokoto.cn/?encode=js&select=%23hitokoto" defer></script>-->
    <!-- 打字特效 -->
    <script src="https://cdn.jsdelivr.net/npm/typed.js@2.0.11"></script>
    <div style="text-align: center;font-weight: bold;color: #1BC3FB;">
      <span id="subtitle"></span>
      <span id="typed-cursor"></span>
    </div>
    <script>
      var typed = new Typed("#subtitle", { strings: ["Live a good life, write some good code !!!", "愿自己的努力终将获得回报。", "花开不是为了花落，而是为了开的更加灿烂。", "没有伞的孩子必须努力奔跑！", "欲望以提升热忱，毅力以磨平高山。", "如果放弃太早，你永远都不知道自己会错过什么。", "没有礁石，就没有美丽的浪花；没有挫折，就没有壮丽的人生。"], startDelay: 1000, typeSpeed: 100, loop: !0, backSpeed: 60, backDelay: 2000, showCursor: !0 })
    </script>
  </cover>
  <header class="l_header pure">
  <div id="loading-bar-wrapper">
    <div id="loading-bar" class="pure"></div>
  </div>

	<div class='wrapper'>
		<div class="nav-main container container--flex">
      <a class="logo flat-box" href='/' >
        
          TRHX'S BLOG
        
      </a>
			<div class='menu navgation'>
				<ul class='h-list'>
          
  					
  						<li>
								<a class="nav flat-box" href="/"
                  
                  
                  id="home">
									<i class='fas fa-home fa-fw'></i>&nbsp;主页
								</a>
							</li>
      			
  						<li>
								<a class="nav flat-box" href="/archives/"
                  
                  
                  id="archives">
									<i class='fas fa-archive fa-fw'></i>&nbsp;归档
								</a>
							</li>
      			
  						<li>
								<a class="nav flat-box" href="/friends/"
                  
                  
                  id="friends">
									<i class='fas fa-users fa-fw'></i>&nbsp;朋友
								</a>
							</li>
      			
  						<li>
								<a class="nav flat-box" href="/comments/"
                  
                  
                  id="comments">
									<i class='fas fa-comments fa-fw'></i>&nbsp;留言
								</a>
							</li>
      			
  						<li>
								<a class="nav flat-box" href="/about/"
                  
                  
                    target="_blank"
                  
                  id="about">
									<i class='fas fa-info-circle fa-fw'></i>&nbsp;关于
								</a>
							</li>
      			
  						<li>
								<a class="nav flat-box" href="/box/"
                  
                  
                    target="_blank"
                  
                  id="box">
									<i class='fas fa-tools fa-fw'></i>&nbsp;百宝箱
								</a>
							</li>
      			
  						<li>
								<a class="nav flat-box" href="https://mi.aliyun.com/shop/40012"
                  
                  
                    target="_blank"
                  
                  id="https:mi.aliyun.comshop40012">
									<i class='fas fa-link fa-fw'></i>&nbsp;米店
								</a>
							</li>
      			
      		
				</ul>
			</div>

			
				<div class="m_search">
					<form name="searchform" class="form u-search-form">
						<input type="text" class="input u-search-input" placeholder="搜索" />
						<i class="icon fas fa-search fa-fw"></i>
					</form>
				</div>
			
			<ul class='switcher h-list'>
				
					<li class='s-search'><a class="fas fa-search fa-fw" href='javascript:void(0)'></a></li>
				
				<li class='s-menu'><a class="fas fa-bars fa-fw" href='javascript:void(0)'></a></li>
			</ul>
		</div>

		<div class='nav-sub container container--flex'>
			<a class="logo flat-box"></a>
			<ul class='switcher h-list'>
				<li class='s-comment'><a class="flat-btn fas fa-comments fa-fw" href='javascript:void(0)'></a></li>
        
          <li class='s-toc'><a class="flat-btn fas fa-list fa-fw" href='javascript:void(0)'></a></li>
        
			</ul>
		</div>
	</div>
</header>
	<aside class="menu-phone">
    <header>
		<nav class="menu navgation">
      <ul>
        
          
            <li>
							<a class="nav flat-box" href="/"
                
                
                id="home">
								<i class='fas fa-home fa-fw'></i>&nbsp;主页
							</a>
            </li>
          
            <li>
							<a class="nav flat-box" href="/archives/"
                
                
                id="archives">
								<i class='fas fa-archive fa-fw'></i>&nbsp;归档
							</a>
            </li>
          
            <li>
							<a class="nav flat-box" href="/friends/"
                
                
                id="friends">
								<i class='fas fa-users fa-fw'></i>&nbsp;朋友
							</a>
            </li>
          
            <li>
							<a class="nav flat-box" href="/comments/"
                
                
                id="comments">
								<i class='fas fa-comments fa-fw'></i>&nbsp;留言
							</a>
            </li>
          
            <li>
							<a class="nav flat-box" href="https://mi.aliyun.com/shop/40012"
                
                
                id="https:mi.aliyun.comshop40012">
								<i class='fas fa-link fa-fw'></i>&nbsp;米店
							</a>
            </li>
          
            <li>
							<a class="nav flat-box" href="/box/"
                
                
                id="box">
								<i class='fas fa-tools fa-fw'></i>&nbsp;百宝箱
							</a>
            </li>
          
            <li>
							<a class="nav flat-box" href="https://itrhx.blog.csdn.net/"
                
                
                id="https:itrhx.blog.csdn.net">
								<i class='fab fa-cuttlefish fa-fw'></i>&nbsp;CSDN
							</a>
            </li>
          
       
      </ul>
		</nav>
    </header>
	</aside>
<script>setLoadingBarProgress(40);</script>

</div>

  <div class="l_body">
    <div class='body-wrapper'>
      <div class='l_main'>
  

    <article id="post" class="post white-box article-type-post" itemscope itemprop="blogPost">
      

  <section class='meta'>
    
    
      <a title='Python3 爬虫学习笔记 C01' href='/2019/08/23/A29-Python3-spider-C01/'><img class='thumbnail' src='https://cdn.jsdelivr.net/gh/TRHX/ImageHosting/ITRHX-PIC/thumbnail/spider.png'></a>
    
    <div class="meta" id="header-meta">
      
        
    <h1 class="title">
      <a href="/2019/08/23/A29-Python3-spider-C01/">
        Python3 爬虫学习笔记 C01
      </a>
    </h1>
  

      <div class='new-meta-box'>
        
          
  <div class='new-meta-item author'>
    <a href="https://www.itrhx.com" rel="nofollow">
      
        <img src="https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@2.1.9/images/trhx.png">
      
      <p>TRHX</p>
    </a>
  </div>


            <div class="new-meta-item date">
  <a class='notlink'>
    <i class="fas fa-calendar-alt" aria-hidden="true"></i>
    <p>2019-08-23</p>
  </a>
</div>

          
  <div class='new-meta-item category'>
    <a href='/categories/Python3-学习笔记/爬虫学习/' rel="nofollow">
      <i class="fas fa-folder-open" aria-hidden="true"></i>
      <p>Python3 学习笔记&nbsp;/&nbsp;爬虫学习</p>
    </a>
  </div>


    <div class="new-meta-item browse busuanzi">
      <a class='notlink'>
        <i class="fas fa-eye" aria-hidden="true"></i>
        <p>
          <span id="busuanzi_value_page_pv">
            <i class="fas fa-spinner fa-spin fa-fw" aria-hidden="true"></i>
          </span>
        </p>
      </a>
    </div>
  

    <div class="new-meta-item wordcount">
      <a class='notlink'>
        <i class="fas fa-keyboard" aria-hidden="true"></i>
        <p>字数统计:</p>
        <p>3,667字</p>
      </a>
    </div>
    <div class="new-meta-item readtime">
      <a class='notlink'>
        <i class="fas fa-hourglass-half" aria-hidden="true"></i>
        <p>阅读时长≈</p>
        <p>17分</p>
      </a>
    </div>
  

      </div>
      
        <hr>
      
    </div>
  </section>


      <section class="article typo">
        <div class="article-entry" itemprop="articleBody">
          <blockquote>
<center><font color="#1BC3FB" size="4">Python3 爬虫学习笔记第一章 ——【基本库 urllib 的使用】</font></center>
</blockquote>
<a id="more"></a>
<h1 id="【1-1】-urllib-简介"><a href="#【1-1】-urllib-简介" class="headerlink" title=" 【1.1】 urllib 简介"></a><font color="#FF0000"> 【1.1】 urllib 简介</font></h1><p>在 Python 中有两种方式可以发送 HTTP 请求，分别是自带的 urllib 库和第三方的 requests 库</p>
<blockquote>
<p>urllib 库：Python 内置的 HTTP 请求库，无需额外安装即可使用；Python 2 中有 urllib 和 urllib2 两个库来实现请求的发送，Python 3 中统一为 urllib。官方文档：<a href="https://docs.python.org/3/library/urllib.html" target="_blank" rel="noopener">https://docs.python.org/3/library/urllib.html</a></p>
</blockquote>
<font color="#FF0000">urllib 所包含的常用模块：</font>

<ul>
<li>urllib.request：模拟发送请求；</li>
<li>urllib.error：异常处理模块，用于捕获异常；</li>
<li>urllib.parse：解析、拆分、合并URL；</li>
<li>urllib.robotparser：读取网站的 robots.txt 文件，判断哪些内容可以爬取。</li>
</ul>
<font color="#FF0000">urllib.request 所包含的常用方法：</font>

<ul>
<li>urllib.request.urlopen()：打开网址URL，这可以是一个字符串或一个 Request对象；</li>
<li>urllib.request.Request()：在请求的时候传入一些 headers 等信息；</li>
<li>urllib.request.urlretrieve()：将获取的URL的内容写到文件目录中去。</li>
</ul>
<font color="#FF0000">urllib.error 所包含的两个异常：</font>

<ul>
<li>URLError：继承自 OSError 类，是 error 异常模块的基类，由 request 模块产生的异常都可以通过捕获这个类来处理。</li>
<li>HTTPError：是 URLError 的子类，专门用来处理 HTTP 请求错误，比如认证请求失败等。</li>
</ul>
<font color="#FF0000">urllib.parse 所包含的常用方法：</font>

<ul>
<li>urllib.parse.urlencode()：将字典参数序列化为 GET 请求参数；</li>
<li>urllib.parse.parse_qs()：将 GET 请求参数反序列化转回字典；</li>
<li>urllib.parse.parse_qsl()：将参数转化为元组组成的列表；</li>
<li>urllib.parse.urlparse()：对 URL 进行分段（返回6个结果）；</li>
<li>urllib.parse.urlunparse()：对 URL 进行组合（长度必须为6）；</li>
<li>urllib.parse.urlsplit()：对 URL 进行分段（不单独解析params部分，返回5个结果）；</li>
<li>urllib.parse.urlunsplit()：对 URL 进行组合（长度必须为5）；</li>
<li>urllib.parse.urljoin()：对 URL 进行组合（没有长度限制，给定两个参数，自动分析 scheme、netloc 和 path 这 3 个内容并对新链接缺失的部分进行补充，最后返回结果）；</li>
<li>urllib.parse.quote()：将内容转化为 URL 编码格式；</li>
<li>urllib.parse.unquote()：对 URL 进行解码。</li>
</ul>
<font color="#FF0000">urllib.robotparser 所包含的类：</font>

<ul>
<li>RobotFileParser：根据网站的 robots.txt 文件来判断一个爬取爬虫是否有权限来爬取这个网页</li>
</ul>
<h1 id="【1-2】-urllib-request-发送请求"><a href="#【1-2】-urllib-request-发送请求" class="headerlink" title=" 【1.2】 urllib.request 发送请求"></a><font color="#FF0000"> 【1.2】 urllib.request 发送请求</font></h1><h2 id="【1-2-1】-urllib-request-urlopen"><a href="#【1-2-1】-urllib-request-urlopen" class="headerlink" title="【1.2.1】 urllib.request.urlopen()"></a><font color="#FF0000">【1.2.1】 urllib.request.urlopen()</font></h2><h3 id="【1-2-1-1】-基本使用方法"><a href="#【1-2-1-1】-基本使用方法" class="headerlink" title="【1.2.1.1】 基本使用方法"></a><font color="#FF0000">【1.2.1.1】 基本使用方法</font></h3><p>urlopen() 函数的 API：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">urllib.request.urlopen(url, data=<span class="keyword">None</span>, [timeout,]*, cafile=<span class="keyword">None</span>, capath=<span class="keyword">None</span>, cadefault=<span class="keyword">False</span>, context=<span class="keyword">None</span>)</span><br></pre></td></tr></table></figure></p>
<p>基本使用：运行以下代码可得到 <a href="https://www.itrhx.com/">https://www.itrhx.com/</a> 的网页源代码：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> urllib.request</span><br><span class="line"></span><br><span class="line">response = urllib.request.urlopen(<span class="string">'https://www.itrhx.com/'</span>)</span><br><span class="line">print(response.read().decode(<span class="string">'utf-8'</span>))</span><br></pre></td></tr></table></figure></p>
<p>输出响应对象的类型和属性：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> urllib.request</span><br><span class="line"></span><br><span class="line">response = urllib.request.urlopen(<span class="string">'https://www.itrhx.com/'</span>)</span><br><span class="line">print(type(response))            <span class="comment"># 响应类型</span></span><br><span class="line">print(response.status)           <span class="comment"># 返回结果的状态码，200代表请求成功</span></span><br><span class="line">print(response.getheaders())       <span class="comment"># 响应的头信息</span></span><br><span class="line">print(response.getheader(<span class="string">'Server'</span>))  <span class="comment"># 获取响应头的 server 值</span></span><br></pre></td></tr></table></figure></p>
<p>运行结果：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">&lt;<span class="class"><span class="keyword">class</span> '<span class="title">http</span>.<span class="title">client</span>.<span class="title">HTTPResponse</span>'&gt;</span></span><br><span class="line"><span class="class">200</span></span><br><span class="line">[('Content-Type', 'text/html; charset=utf-8'), ('Server', 'GitHub.com'), ('Last-Modified', 'Sat, 17 Aug 2019 12:16:48 GMT'), ('ETag', '"5d57f030-10863"'), ('Access-Control-Allow-Origin', '*'), ('Expires', 'Sat, 17 Aug 2019 19:41:25 GMT'), ('Cache-Control', 'max-age=600'), ('X-Proxy-Cache', 'MISS'), ('X-GitHub-Request-Id', 'C748:735D:5B7461:619B95:5D58560B'), ('Content-Length', '67683'), ('Accept-Ranges', 'bytes'), ('Date', 'Sun, 18 Aug 2019 13:28:44 GMT'), ('Via', '1.1 varnish'), ('Age', '228'), ('Connection', 'close'), ('X-Served-By', 'cache-tyo19931-TYO'), ('X-Cache', 'HIT'), ('X-Cache-Hits', '1'), ('X-Timer', 'S1566134924.190474,VS0,VE0'), ('Vary', 'Accept-Encoding'), ('X-Fastly-Request-ID', '25a69f8130fc9cae412d28990a724543d7d05e8b')]</span><br><span class="line">GitHub.com</span><br></pre></td></tr></table></figure></p>
<h3 id="【1-2-1-2】-添加参数"><a href="#【1-2-1-2】-添加参数" class="headerlink" title="【1.2.1.2】 添加参数"></a><font color="#FF0000">【1.2.1.2】 添加参数</font></h3><p>根据 urlopen() 函数的 API 可知，除了最基本的 URL 参数以外，我们还可以传递其他内容，比如 data（附加数据）、timeout（超时时间）等，以下用 data 和 timeout 参数举例说明。</p>
<h4 id="●-data-参数"><a href="#●-data-参数" class="headerlink" title="● data 参数"></a><font color="#FF0000">● data 参数</font></h4><p>如果要添加 data 参数，需要使用 bytes 方法将参数转化为字节流编码格式的内容，即 bytes 类型。另外，如果传递了这个参数，则它的请求方式就不再是 GET 方式，而是 POST 方式。代码示例：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> urllib.parse</span><br><span class="line"><span class="keyword">import</span> urllib.request</span><br><span class="line"></span><br><span class="line">data = bytes(urllib.parse.urlencode(&#123;<span class="string">'word'</span>: <span class="string">'hello'</span>&#125;), encoding=<span class="string">'utf8'</span>)</span><br><span class="line">response = urllib.request.urlopen(<span class="string">'http://httpbin.org/post'</span>, data=data)</span><br><span class="line">print(response.read())</span><br></pre></td></tr></table></figure></p>
<p>httpbin.org 站点提供 HTTP 请求测试，<a href="http://httpbin.org/post" target="_blank" rel="noopener">http://httpbin.org/post</a> 用于测试 POST 请求，示例中传递一个值为 hello 的 word 参数。使用 bytes 方法，将其转码成 bytes（字节流）类型。该方法的第一个参数需要是 str（字符串）类型，需要用 urllib.parse 模块里的 urlencode 方法来将参数字典转化为字符串；第二个参数指定编码格式为 utf8，运行结果：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line"><span class="string">b'&#123;</span></span><br><span class="line"><span class="string">	"args": &#123;&#125;,</span></span><br><span class="line"><span class="string">	"data": "", </span></span><br><span class="line"><span class="string">	"files": &#123;&#125;,</span></span><br><span class="line"><span class="string">	"form": &#123;</span></span><br><span class="line"><span class="string">		"word": "hello"</span></span><br><span class="line"><span class="string">	&#125;,</span></span><br><span class="line"><span class="string">	"headers": &#123;</span></span><br><span class="line"><span class="string">		"Accept-Encoding": "identity", </span></span><br><span class="line"><span class="string">		"Content-Length": "10",</span></span><br><span class="line"><span class="string">		"Content-Type": "application/x-www-form-urlencoded",</span></span><br><span class="line"><span class="string">		"Host": "httpbin.org",</span></span><br><span class="line"><span class="string">		"User-Agent": "Python-urllib/3.6"</span></span><br><span class="line"><span class="string">	&#125;, </span></span><br><span class="line"><span class="string">	"json": null, </span></span><br><span class="line"><span class="string">	"origin": "171.115.101.10, 171.115.101.10", </span></span><br><span class="line"><span class="string">	"url": "https://httpbin.org/post"</span></span><br><span class="line"><span class="string">&#125;'</span></span><br></pre></td></tr></table></figure></p>
<h4 id="●-timeout-参数"><a href="#●-timeout-参数" class="headerlink" title="● timeout 参数"></a><font color="#FF0000">● timeout 参数</font></h4><p>举例：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> urllib.request</span><br><span class="line"></span><br><span class="line">response = urllib.request.urlopen(<span class="string">'http://httpbin.org/get'</span>, timeout=<span class="number">0.1</span>)  </span><br><span class="line">print(response.read())</span><br></pre></td></tr></table></figure></p>
<p>运行结果：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">...</span><br><span class="line">During handling of the above exception, another exception occurred:</span><br><span class="line"></span><br><span class="line">Traceback (most recent call last):</span><br><span class="line">  File <span class="string">"C:/Users/Lenovo/Desktop/1.py"</span>, line <span class="number">2</span>, <span class="keyword">in</span> &lt;module&gt;</span><br><span class="line">    response = urllib.request.urlopen(<span class="string">'http://httpbin.org/get'</span>, timeout=<span class="number">0.1</span>)</span><br><span class="line"> ...</span><br><span class="line">urllib.error.URLError: &lt;urlopen error timed out&gt;</span><br></pre></td></tr></table></figure></p>
<p>timeout 设置为0.1，0.1秒过后服务器没有响应，便会抛出 URLError 异常<br>进阶：使用 try except 语句抛出异常</p>
<h2 id="【1-2-2】-urllib-request-Request"><a href="#【1-2-2】-urllib-request-Request" class="headerlink" title="【1.2.2】 urllib.request.Request()"></a><font color="#FF0000">【1.2.2】 urllib.request.Request()</font></h2><p>Request() 方法可以在请求的时候传入一些 data、headers 等信息<br>Request() 的构造方法：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">urllib</span>.<span class="title">request</span>.<span class="title">Request</span><span class="params">(url, data=None, headers=&#123;&#125;, origin_req_host=None, unverifiable=False, method=None)</span></span></span><br></pre></td></tr></table></figure></p>
<p>构造方法各个参数的解释：</p>
<ul>
<li><p>url：用于请求 URL，这是必传参数，其他都是可选参数。</p>
</li>
<li><p>data：如果要传，必须传 bytes（字节流）类型的。如果它是字典，可以先用 urllib.parse 模块里的 urlencode() 编码。</p>
</li>
<li><p>headers：是一个字典，它就是请求头，可以在构造请求时通过 headers 参数直接构造，也可以通过调用请求实例的 add_header() 方法添加。添加请求头最常用的用法就是通过修改 User-Agent 来伪装浏览器，默认的 User-Agent 是 Python-urllib，我们可以通过修改它来伪装浏览器。</p>
</li>
<li><p>origin_req_host：指的是请求方的 host 名称或者 IP 地址。</p>
</li>
<li><p>unverifiable：表示这个请求是否是无法验证的，默认是 False，意思就是说用户没有足够权限来选择接收这个请求的结果。例如，我们请求一个 HTML 文档中的图片，但是我们没有自动抓取图像的权限，这时 unverifiable 的值就是 True。</p>
</li>
<li><p>method：是一个字符串，用来指示请求使用的方法，比如 GET、POST 和 PUT 等。</p>
</li>
</ul>
<p>简单举例：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> urllib.request</span><br><span class="line"><span class="keyword">import</span> urllib.parse</span><br><span class="line"></span><br><span class="line">url = <span class="string">'http://www.baidu.com/'</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 定制要伪装的头部</span></span><br><span class="line">headers = &#123;</span><br><span class="line">    <span class="string">'User-Agent'</span>: <span class="string">'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'</span></span><br><span class="line">&#125;</span><br><span class="line"><span class="comment"># 构建请求对象</span></span><br><span class="line">request = urllib.request.Request(url=url, headers=headers)</span><br><span class="line"><span class="comment"># 发送请求</span></span><br><span class="line">response = urllib.request.urlopen(request)</span><br><span class="line">print(response.read().decode())</span><br></pre></td></tr></table></figure></p>
<h2 id="【1-2-3】-urllib-request-urlretrieve"><a href="#【1-2-3】-urllib-request-urlretrieve" class="headerlink" title="【1.2.3】 urllib.request.urlretrieve()"></a><font color="#FF0000">【1.2.3】 urllib.request.urlretrieve()</font></h2><p>将获取到的 URL 内容保存到当前文件夹，简单举例：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> urllib.request</span><br><span class="line"></span><br><span class="line">url = <span class="string">'https://www.itrhx.com/images/trhx.png'</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># response = urllib.request.urlopen(image_url)</span></span><br><span class="line"><span class="comment"># with open('trhx.png', 'wb') as fp:</span></span><br><span class="line"><span class="comment">#    fp.write(response.read())</span></span><br><span class="line"></span><br><span class="line">urllib.request.urlretrieve(url, <span class="string">'trhx.png'</span>)</span><br></pre></td></tr></table></figure></p>
<h1 id="【1-3】-urllib-error-异常处理"><a href="#【1-3】-urllib-error-异常处理" class="headerlink" title=" 【1.3】 urllib.error 异常处理"></a><font color="#FF0000"> 【1.3】 urllib.error 异常处理</font></h1><h2 id="【1-3-1】-URLError"><a href="#【1-3-1】-URLError" class="headerlink" title="【1.3.1】 URLError"></a><font color="#FF0000">【1.3.1】 URLError</font></h2><p>如果打开一个不存在的页面，就会出现 URLError 错误，该错误有一个 reason 属性，用于返回错误的原因。简单举例：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> urllib <span class="keyword">import</span> request, error  </span><br><span class="line"><span class="keyword">try</span>:  </span><br><span class="line">    response = request.urlopen(<span class="string">'https://www.itrhx.com/index/'</span>)  </span><br><span class="line"><span class="keyword">except</span> error.URLError <span class="keyword">as</span> e:  </span><br><span class="line">    print(e.reason)</span><br></pre></td></tr></table></figure></p>
<p>输出结果：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">Not Found</span><br></pre></td></tr></table></figure></p>
<h2 id="【1-3-2】-HTTPError"><a href="#【1-3-2】-HTTPError" class="headerlink" title="【1.3.2】 HTTPError"></a><font color="#FF0000">【1.3.2】 HTTPError</font></h2><p>URLError 的子类，专门用来处理 HTTP 请求错误，比如认证请求失败等。它有如下3个属性：</p>
<ul>
<li>code：返回 HTTP 状态码，比如 404 表示网页不存在，500 表示服务器内部错误等。</li>
<li>reason：同父类一样，用于返回错误的原因。</li>
<li>headers：返回请求头。</li>
</ul>
<p>简单举例：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> urllib <span class="keyword">import</span> request, error  </span><br><span class="line"><span class="keyword">try</span>:  </span><br><span class="line">    response = request.urlopen(<span class="string">'https://www.itrhx.com/index/'</span>)  </span><br><span class="line"><span class="keyword">except</span> error.HTTPError <span class="keyword">as</span> e:  </span><br><span class="line">    print(e.code, e.reason, e.headers)</span><br></pre></td></tr></table></figure></p>
<p>输出结果：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line"><span class="number">404</span> Not Found Content-Type: text/html; charset=utf<span class="number">-8</span></span><br><span class="line">Server: GitHub.com</span><br><span class="line">ETag: <span class="string">"5d57f030-7f2"</span></span><br><span class="line">Access-Control-Allow-Origin: *</span><br><span class="line">X-Proxy-Cache: MISS</span><br><span class="line">X-GitHub-Request-Id: <span class="number">4</span>B46:<span class="number">2</span>F5D:<span class="number">6</span>DE0F1:<span class="number">755</span>BB2:<span class="number">5</span>D5964C5</span><br><span class="line">Content-Length: <span class="number">2034</span></span><br><span class="line">Accept-Ranges: bytes</span><br><span class="line">Date: Sun, <span class="number">18</span> Aug <span class="number">2019</span> <span class="number">14</span>:<span class="number">50</span>:<span class="number">41</span> GMT</span><br><span class="line">Via: <span class="number">1.1</span> varnish</span><br><span class="line">Age: <span class="number">252</span></span><br><span class="line">Connection: close</span><br><span class="line">X-Served-By: cache-tyo19951-TYO</span><br><span class="line">X-Cache: HIT</span><br><span class="line">X-Cache-Hits: <span class="number">1</span></span><br><span class="line">X-Timer: S1566139842<span class="number">.563134</span>,VS0,VE0</span><br><span class="line">Vary: Accept-Encoding</span><br><span class="line">X-Fastly-Request-ID: e9eb0a507be66a866bfaa7c5cc2e1c53b1f7ccab</span><br></pre></td></tr></table></figure></p>
<h2 id="【1-3-3】-进阶用法"><a href="#【1-3-3】-进阶用法" class="headerlink" title="【1.3.3】 进阶用法"></a><font color="#FF0000">【1.3.3】 进阶用法</font></h2><p>因为 URLError 是 HTTPError 的父类，所以可以先选择捕获子类的错误，再去捕获父类的错误，前面的代码改进：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> urllib <span class="keyword">import</span> request, error  </span><br><span class="line"></span><br><span class="line"><span class="keyword">try</span>:  </span><br><span class="line">    response = request.urlopen(<span class="string">'https://www.itrhx.com/index/'</span>)  </span><br><span class="line"><span class="keyword">except</span> error.HTTPError <span class="keyword">as</span> e:  </span><br><span class="line">    print(e.reason, e.code, e.headers)  </span><br><span class="line"><span class="keyword">except</span> error.URLError <span class="keyword">as</span> e:  </span><br><span class="line">    print(e.reason)  </span><br><span class="line"><span class="keyword">else</span>:  </span><br><span class="line">    print(<span class="string">'Request Successfully'</span>)</span><br></pre></td></tr></table></figure></p>
<h1 id="【1-4】-urllib-parse-解析-URL"><a href="#【1-4】-urllib-parse-解析-URL" class="headerlink" title=" 【1.4】 urllib.parse 解析 URL"></a><font color="#FF0000"> 【1.4】 urllib.parse 解析 URL</font></h1><h2 id="【1-4-1】-urllib-parse-urlencode"><a href="#【1-4-1】-urllib-parse-urlencode" class="headerlink" title="【1.4.1】 urllib.parse.urlencode()"></a><font color="#FF0000">【1.4.1】 urllib.parse.urlencode()</font></h2><p>将字典参数序列化为 GET 请求参数，示例：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> urllib.parse <span class="keyword">import</span> urlencode</span><br><span class="line">data = &#123;</span><br><span class="line">    <span class="string">'ie'</span>: <span class="string">'utf-8'</span>,</span><br><span class="line">    <span class="string">'wd'</span>: <span class="string">'TRHX'</span>,</span><br><span class="line">&#125;</span><br><span class="line">base_url = <span class="string">'http://www.baidu.com?'</span></span><br><span class="line">url = base_url + urlencode(data)</span><br><span class="line">print(url)</span><br></pre></td></tr></table></figure></p>
<p>输出结果：<br><figure class="highlight"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">http://www.baidu.com?ie=utf-8&amp;wd=TRHX</span><br></pre></td></tr></table></figure></p>
<h2 id="【1-4-2】-urllib-parse-parse-qs"><a href="#【1-4-2】-urllib-parse-parse-qs" class="headerlink" title="【1.4.2】 urllib.parse.parse_qs()"></a><font color="#FF0000">【1.4.2】 urllib.parse.parse_qs()</font></h2><p>与 urlencode() 相反，将 GET 请求参数反序列化转回字典，示例：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> urllib.parse <span class="keyword">import</span> parse_qs</span><br><span class="line">query = <span class="string">'name=TRHX&amp;age=20'</span></span><br><span class="line">print(parse_qs(query))</span><br></pre></td></tr></table></figure></p>
<p>输出结果：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">&#123;<span class="string">'name'</span>: [<span class="string">'TRHX'</span>], <span class="string">'age'</span>: [<span class="string">'20'</span>]&#125;</span><br></pre></td></tr></table></figure></p>
<h2 id="【1-4-3】-urllib-parse-parse-qsl"><a href="#【1-4-3】-urllib-parse-parse-qsl" class="headerlink" title="【1.4.3】 urllib.parse.parse_qsl()"></a><font color="#FF0000">【1.4.3】 urllib.parse.parse_qsl()</font></h2><p>将参数转化为元组组成的列表，示例：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> urllib.parse <span class="keyword">import</span> parse_qsl</span><br><span class="line">query = <span class="string">'name=TRHX&amp;age=20'</span></span><br><span class="line">print(parse_qsl(query))</span><br></pre></td></tr></table></figure></p>
<p>输出 结果：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">[(<span class="string">'name'</span>, <span class="string">'TRHX'</span>), (<span class="string">'age'</span>, <span class="string">'20'</span>)]</span><br></pre></td></tr></table></figure></p>
<h2 id="【1-4-4】-urllib-parse-urlparse"><a href="#【1-4-4】-urllib-parse-urlparse" class="headerlink" title="【1.4.4】 urllib.parse.urlparse()"></a><font color="#FF0000">【1.4.4】 urllib.parse.urlparse()</font></h2><p>对 URL 进行分段，返回 6 个结果，示例：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> urllib.parse <span class="keyword">import</span> urlparse</span><br><span class="line">result = urlparse(<span class="string">'http://www.baidu.com/index.html;user?id=5#comment'</span>)</span><br><span class="line">print(type(result), result)</span><br></pre></td></tr></table></figure></p>
<p>输出结果：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">&lt;<span class="class"><span class="keyword">class</span> '<span class="title">urllib</span>.<span class="title">parse</span>.<span class="title">ParseResult</span>'&gt; <span class="title">ParseResult</span><span class="params">(scheme=<span class="string">'http'</span>, netloc=<span class="string">'www.baidu.com'</span>, path=<span class="string">'/index.html'</span>, params=<span class="string">'user'</span>, query=<span class="string">'id=5'</span>, fragment=<span class="string">'comment'</span>)</span></span></span><br></pre></td></tr></table></figure></p>
<p>返回结果为 ParseResult 类型的对象，含 scheme、netloc、path、params、query 和 fragment 6 个部分，依次代表协议、域名、路径、参数、查询条件、锚点</p>
<h2 id="【1-4-5】-urllib-parse-urlunparse"><a href="#【1-4-5】-urllib-parse-urlunparse" class="headerlink" title="【1.4.5】 urllib.parse.urlunparse()"></a><font color="#FF0000">【1.4.5】 urllib.parse.urlunparse()</font></h2><p>与 urlparse() 相反，对 URL 进行组合，传入的参数是一个可迭代对象，长度必须是 6，否则会抛出参数数量不足或者过多的问题，示例：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> urllib.parse <span class="keyword">import</span> urlunparse  </span><br><span class="line">data = [<span class="string">'http'</span>, <span class="string">'www.baidu.com'</span>, <span class="string">'index.html'</span>, <span class="string">'user'</span>, <span class="string">'a=6'</span>, <span class="string">'comment'</span>]  </span><br><span class="line">print(urlunparse(data))</span><br></pre></td></tr></table></figure></p>
<p>输出结果：<br><figure class="highlight"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">http://www.baidu.com/index.html;user?a=6#comment</span><br></pre></td></tr></table></figure></p>
<h2 id="【1-4-6】-urllib-parse-urlsplit"><a href="#【1-4-6】-urllib-parse-urlsplit" class="headerlink" title="【1.4.6】 urllib.parse.urlsplit()"></a><font color="#FF0000">【1.4.6】 urllib.parse.urlsplit()</font></h2><p>与 urlparse() 方法相似，但是它不再单独解析 params 部分，只返回 5 个结果。params 会合并到 path 中，示例：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> urllib.parse <span class="keyword">import</span> urlsplit  </span><br><span class="line">result = urlsplit(<span class="string">'http://www.baidu.com/index.html;user?id=5#comment'</span>)  </span><br><span class="line">print(result)</span><br></pre></td></tr></table></figure></p>
<p>输出结果：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">SplitResult(scheme=<span class="string">'http'</span>, netloc=<span class="string">'www.baidu.com'</span>, path=<span class="string">'/index.html;user'</span>, query=<span class="string">'id=5'</span>, fragment=<span class="string">'comment'</span>)</span><br></pre></td></tr></table></figure></p>
<h2 id="【1-4-7】-urllib-parse-urlunsplit"><a href="#【1-4-7】-urllib-parse-urlunsplit" class="headerlink" title="【1.4.7】 urllib.parse.urlunsplit()"></a><font color="#FF0000">【1.4.7】 urllib.parse.urlunsplit()</font></h2><p>与 urlunparse() 方法类似，对 URL 进行组合，传入的参数也是一个可迭代对象，长度必须为 5，示例：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> urllib.parse <span class="keyword">import</span> urlunsplit  </span><br><span class="line">data = [<span class="string">'http'</span>, <span class="string">'www.baidu.com'</span>, <span class="string">'index.html'</span>, <span class="string">'a=6'</span>, <span class="string">'comment'</span>]  </span><br><span class="line">print(urlunsplit(data))</span><br></pre></td></tr></table></figure></p>
<p>输出结果：<br><figure class="highlight"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">http://www.baidu.com/index.html?a=6#comment</span><br></pre></td></tr></table></figure></p>
<h2 id="【1-4-8】-urllib-parse-urljoin"><a href="#【1-4-8】-urllib-parse-urljoin" class="headerlink" title="【1.4.8】 urllib.parse.urljoin()"></a><font color="#FF0000">【1.4.8】 urllib.parse.urljoin()</font></h2><p>对 URL 进行组合，提供两个 URL 作为两个参数，将会自动分析 URL 的 scheme、netloc 和 path 这 3 个内容并对新链接缺失的部分进行补充，最后返回结果，示例：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> urllib.parse <span class="keyword">import</span> urljoin  </span><br><span class="line">print(urljoin(<span class="string">'http://www.baidu.com'</span>, <span class="string">'friends.html'</span>))  </span><br><span class="line">print(urljoin(<span class="string">'http://www.baidu.com'</span>, <span class="string">'https://www.itrhx.com/friends.html'</span>))  </span><br><span class="line">print(urljoin(<span class="string">'http://www.baidu.com/friends.html'</span>, <span class="string">'https://www.itrhx.com/friends.html'</span>))  </span><br><span class="line">print(urljoin(<span class="string">'http://www.baidu.com/friends.html'</span>, <span class="string">'https://www.itrhx.com/friends.html?id=2'</span>))  </span><br><span class="line">print(urljoin(<span class="string">'http://www.baidu.com?wd=trhx'</span>, <span class="string">'https://www.itrhx.com/index.html'</span>))  </span><br><span class="line">print(urljoin(<span class="string">'http://www.baidu.com'</span>, <span class="string">'?category=2#comment'</span>))  </span><br><span class="line">print(urljoin(<span class="string">'www.baidu.com'</span>, <span class="string">'?category=2#comment'</span>))  </span><br><span class="line">print(urljoin(<span class="string">'www.baidu.com#comment'</span>, <span class="string">'?category=2'</span>))</span><br></pre></td></tr></table></figure></p>
<p>输出结果：<br><figure class="highlight"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">http://www.baidu.com/friends.html</span><br><span class="line">https://www.itrhx.com/friends.html</span><br><span class="line">https://www.itrhx.com/friends.html</span><br><span class="line">https://www.itrhx.com/friends.html?id=2</span><br><span class="line">https://www.itrhx.com/index.html</span><br><span class="line">http://www.baidu.com?category=2#comment</span><br><span class="line">www.baidu.com?category=2#comment</span><br><span class="line">www.baidu.com?category=2</span><br></pre></td></tr></table></figure></p>
<h2 id="【1-4-9】-urllib-parse-quote"><a href="#【1-4-9】-urllib-parse-quote" class="headerlink" title="【1.4.9】 urllib.parse.quote()"></a><font color="#FF0000">【1.4.9】 urllib.parse.quote()</font></h2><p>将内容转化为 URL 编码的格式。当 URL 中带有中文参数时，可以将中文字符转化为 URL 编码，示例：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> urllib.parse <span class="keyword">import</span> quote</span><br><span class="line">keyword = <span class="string">'中国'</span>  </span><br><span class="line">url = <span class="string">'https://www.baidu.com/s?wd='</span> + quote(keyword)  </span><br><span class="line">print(url)</span><br></pre></td></tr></table></figure></p>
<p>输出结果：<br><figure class="highlight"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">https://www.baidu.com/s?wd=%E4%B8%AD%E5%9B%BD</span><br></pre></td></tr></table></figure></p>
<h2 id="【1-4-10】-urllib-parse-unquote"><a href="#【1-4-10】-urllib-parse-unquote" class="headerlink" title="【1.4.10】 urllib.parse.unquote()"></a><font color="#FF0000">【1.4.10】 urllib.parse.unquote()</font></h2><p>与 quote() 方法相反，对 URL 进行解码，示例：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> urllib.parse <span class="keyword">import</span> unquote  </span><br><span class="line">url = <span class="string">'https://www.baidu.com/s?wd=%E4%B8%AD%E5%9B%BD'</span>  </span><br><span class="line">print(unquote(url))</span><br></pre></td></tr></table></figure></p>
<p>输出结果：<br><figure class="highlight"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">https://www.baidu.com/s?wd=中国</span><br></pre></td></tr></table></figure></p>
<h1 id="【1-5】-urllib-robotparser-爬取权限判断"><a href="#【1-5】-urllib-robotparser-爬取权限判断" class="headerlink" title=" 【1.5】 urllib.robotparser 爬取权限判断"></a><font color="#FF0000"> 【1.5】 urllib.robotparser 爬取权限判断</font></h1><h2 id="【1-5-1】-Robots-协议简介"><a href="#【1-5-1】-Robots-协议简介" class="headerlink" title="【1.5.1】 Robots 协议简介"></a><font color="#FF0000">【1.5.1】 Robots 协议简介</font></h2><blockquote>
<p>Robots 协议即爬虫协议，用来告诉爬虫和搜索引擎哪些页面可以抓取，哪些不可以抓取。它通常是一个叫作 robots.txt 的文本文件，一般放在网站的根目录下。</p>
</blockquote>
<p>robots.txt 基本格式：<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">User-agent:</span><br><span class="line">Disallow:</span><br><span class="line">Allow:</span><br></pre></td></tr></table></figure></p>
<ul>
<li>User-agent 为搜索爬虫的名称，设置为 * 则表示对任何爬虫皆有效；</li>
<li>Disallow 指定了不允许抓取的目录，设置为 / 则代表不允许抓取所有页面；</li>
<li>Allow 指定了允许抓取的目录，一般和 Disallow 一起使用，一般不会单独使用，用来排除某些限制。</li>
</ul>
<p>一些常见的搜索爬虫名称及其对应的网站：</p>
<table>
<thead>
<tr>
<th>爬虫名称</th>
<th>网站名称</th>
<th>网站地址</th>
</tr>
</thead>
<tbody>
<tr>
<td>BaiduSpider</td>
<td>百度</td>
<td><a href="http://www.baidu.com" target="_blank" rel="noopener">www.baidu.com</a></td>
</tr>
<tr>
<td>Googlebot</td>
<td>谷歌</td>
<td><a href="http://www.google.com" target="_blank" rel="noopener">www.google.com</a></td>
</tr>
<tr>
<td>360Spider</td>
<td>360</td>
<td><a href="http://www.so.com" target="_blank" rel="noopener">www.so.com</a></td>
</tr>
<tr>
<td>Sogouspider</td>
<td>搜狗</td>
<td><a href="http://www.sogou.com" target="_blank" rel="noopener">www.sogou.com</a></td>
</tr>
<tr>
<td>YodaoBot</td>
<td>有道</td>
<td><a href="http://www.youdao.com" target="_blank" rel="noopener">www.youdao.com</a></td>
</tr>
<tr>
<td>Bingbot</td>
<td>必应</td>
<td><a href="http://www.bing.com" target="_blank" rel="noopener">www.bing.com</a></td>
</tr>
<tr>
<td>Yahoo!  Slurp</td>
<td>雅虎</td>
<td><a href="http://www.yahoo.com" target="_blank" rel="noopener">www.yahoo.com</a></td>
</tr>
<tr>
<td>ia_archiver</td>
<td>Alexa</td>
<td><a href="http://www.alexa.cn" target="_blank" rel="noopener">www.alexa.cn</a></td>
</tr>
<tr>
<td>Scooter</td>
<td>altavista</td>
<td><a href="http://www.altavista.com" target="_blank" rel="noopener">www.altavista.com</a></td>
</tr>
</tbody>
</table>
<h2 id="【1-5-2】-RobotFileParser-类常用方法"><a href="#【1-5-2】-RobotFileParser-类常用方法" class="headerlink" title="【1.5.2】 RobotFileParser 类常用方法"></a><font color="#FF0000">【1.5.2】 RobotFileParser 类常用方法</font></h2><p>RobotFileParser 类的声明：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">urllib.robotparser.RobotFileParser(url=<span class="string">''</span>)</span><br></pre></td></tr></table></figure></p>
<p>常用方法及其解释：</p>
<ul>
<li><p>set_url：用来设置 robots.txt 文件的链接。如果在创建 RobotFileParser<br>对象时传入了链接，那么就不需要再用这种方法了。</p>
</li>
<li><p>read：读取 robots.txt 文件并进行分析。此方法执行一个读取和分析操作，若不调用此方法，接下来的判断都会为 False，这个方法不会返回任何内容，但是执行了读取操作。</p>
</li>
<li><p>parse：解析 robots.txt 文件，传入的参数是 robots.txt 某些行的内容，它会按照 robots.txt 的语法规则来分析这些内容。</p>
</li>
<li><p>can_fetch：该方法传入两个参数，第一个是 User-agent，第二个是要抓取的 URL。返回的内容是该搜索引擎是否可以抓取这个 URL，返回结果是 True 或 False。</p>
</li>
<li><p>mtime：返回的是上次抓取和分析 robots.txt 的时间，此方法可以定期检查来抓取最新的 robots.txt。</p>
</li>
<li><p>modified：将当前时间设置为上次抓取和分析 robots.txt 的时间。</p>
</li>
</ul>
<p>以简书为例：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> urllib.robotparser <span class="keyword">import</span> RobotFileParser</span><br><span class="line">rp = RobotFileParser()</span><br><span class="line">rp.set_url(<span class="string">'http://www.jianshu.com/robots.txt'</span>)</span><br><span class="line">rp.read()</span><br><span class="line">print(rp.can_fetch(<span class="string">'*'</span>, <span class="string">'https://www.jianshu.com/p/6d9527300b4c'</span>))</span><br><span class="line">print(rp.can_fetch(<span class="string">'*'</span>, <span class="string">"http://www.jianshu.com/search?q=python&amp;page=1&amp;type=collections"</span>))</span><br></pre></td></tr></table></figure></p>
<p>输出结果：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">False</span></span><br><span class="line"><span class="keyword">False</span></span><br></pre></td></tr></table></figure></p>

        </div>
        
          <div id="reward">
  
  <div>您的喜欢是作者写作最大的动力！❤️</div>
  
  <div class="reward">
    <a href="https://github.com/Kaiyuan/donate-page" target="_blank" class=" tr3" title="Github"><span
        id="github"></span></a>
    <ul id="RewardBox" class="list pos-f tr3">
      
      <li id="PayPal" title="PayPal打赏"><a href="https://www.paypal.me/" target="_blank">PayPal</a></li>
      
      
      <li id="AliPayOR" title="支付宝打赏">AliPay</li>
      
      
      <li id="WeChatPayOR" title="微信打赏">WeChatPay</li>
      
      
      <li id="QQPayOR" title="QQ打赏">QQPay</li>
      
    </ul>
    <div id="RewardText" class="tr3">Donate</div>
    <div id="QRBox" class="pos-f left-100">
      <div id="MainBox"></div>
    </div>
  </div>
</div>
<script src="https://ajax.aspnetcdn.com/ajax/jQuery/jquery-2.0.3.min.js"></script>
<script>
  jQuery(document).ready(function () {
    var QRBox = $('#QRBox');
    var MainBox = $('#MainBox');
    var AliPayOR = 'https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.7/images/reward/AliPayQR.png';
    var WeChatPayOR = 'https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.7/images/reward/WeChatQR.png';
    var QQPayOR = 'https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.7/images/reward/QQPayQR.png';

    function showQR(QR) {
      if (QR) {
        MainBox.css('background-image', 'url(' + QR + ')');
      }
      $('#RewardText,#RewardBox,#github').addClass('blur');
      QRBox.fadeIn(300, function (argument) {
        MainBox.addClass('showQR');
      });
    }

    $('#RewardBox>li').click(function (event) {
      var thisID = $(this).attr('id');
      if (thisID === 'AliPayOR') {
        showQR(AliPayOR);
      } else if (thisID === 'WeChatPayOR') {
        showQR(WeChatPayOR);
      } else if (thisID === 'QQPayOR') {
        showQR(QQPayOR);
      }
    });

    MainBox.click(function (event) {
      MainBox.removeClass('showQR').addClass('hideQR');
      setTimeout(function (a) {
        QRBox.fadeOut(300, function (argument) {
          MainBox.removeClass('hideQR');
        });
        $('#RewardText,#RewardBox,#github').removeClass('blur');
      }, 600);

    });
  });
</script>

        
  <section class='meta' id="footer-meta">
    <hr>
    <div class='new-meta-box'>
      
        
          <div class="new-meta-item date" itemprop="dateUpdated" datetime="2019-09-24T20:39:54+08:00">
  <a class='notlink'>
    <i class="fas fa-clock" aria-hidden="true"></i>
    <p>最后更新于 2019年9月24日</p>
  </a>
</div>

        
  <div class="new-meta-item meta-tags"><a class="tag" href="/tags/爬虫/" rel="nofollow"><i class="fas fa-tags" aria-hidden="true"></i>&nbsp;<p>爬虫</p></a></div> <div class="new-meta-item meta-tags"><a class="tag" href="/tags/urllib/" rel="nofollow"><i class="fas fa-tags" aria-hidden="true"></i>&nbsp;<p>urllib</p></a></div>


  <div class="new-meta-item share -mob-share-list">
  <div class="-mob-share-list share-body">
    
      
        <a class="-mob-share-qq" title="QQ好友" rel="external nofollow noopener noreferrer"
          
          href="http://connect.qq.com/widget/shareqq/index.html?url=https://www.itrhx.com/2019/08/23/A29-Python3-spider-C01/&title=Python3 爬虫学习笔记 C01 | TRHX'S BLOG&pics=https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.4/images/trhx.png&summary=
Python3 爬虫学习笔记第一章 ——【基本库 urllib 的使用】
"
          
          >
          
            <img src="https://cdn.jsdelivr.net/gh/xaoxuu/assets@19.1.9/logo/128/qq.png">
          
        </a>
      
    
        <a class="-mob-share-qzone" title="QQ空间" rel="external nofollow noopener noreferrer"
          
          href="https://sns.qzone.qq.com/cgi-bin/qzshare/cgi_qzshare_onekey?url=https://www.itrhx.com/2019/08/23/A29-Python3-spider-C01/&title=Python3 爬虫学习笔记 C01 | TRHX'S BLOG&pics=https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.4/images/trhx.png&summary=
Python3 爬虫学习笔记第一章 ——【基本库 urllib 的使用】
"
          
          >
          
            <img src="https://cdn.jsdelivr.net/gh/xaoxuu/assets@19.1.9/logo/128/qzone.png">
          
        </a>
      
    
        <a class='qrcode' rel="external nofollow noopener noreferrer" href='data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAN4AAADeCAAAAAB3DOFrAAACu0lEQVR42u3aMW7DQAwEQP//00kbILG9S+piFaMqsCPrRsUdseTjEV9fP67fn/z+9tm9ye+8fuLjxIWHh4e3WHr7sNePTJb47JP86W9eMR4eHt4x3rPFJct6zbj2CMnXjIeHh3cfXnIYJJt+fmzk5TUeHh7ePXlJST0rwfPXioeHh3cHXlvO7j9pA4vjWQseHh5ezMu7SPf5+0h/Dw8PD2/dVZ8Fr20Bnce+9Wrx8PDwDvDyDXczAtUW6MkvFP+Jh4eHd4CXb/GzInhWLm8CCDw8PLxzvISRvIJ9i2t2VESHAR4eHt4xXh4EtAMHbSmcl+B1DIGHh4e35rXLaltTs/B3c8w8NicJHh4eXsBrI9R2y85jhbZwj+IJPDw8vAO8fC/dDxZshqvyQ+jNQYKHh4e35s2aTJuIod36V6MJeHh4eAd4szZY28TaxBbttxd0+fDw8PDSivSCpv4s1NgU022sjIeHh7fnFQFoCc5j37ZML/IVPDw8vGO8TWGdN8naptemAYaHh4f3Wd5mv80XlxTN7Rrw8PDwzvFmQ1TDsae4xXVBsY6Hh4d3gLc5EmatqdmwQhsTF8k0Hh4e3pqXbMGz9tjmiUnZHUUSeHh4eGte23Zqo4E85J1Fw2+OKzw8PLwDvPwAyLf+WSw7G8yKmnZ4eHh4h3l7Rl58J3FtO/61mhTDw8PDK3mzRtR+eGs2cFBEwHh4eHj/wkuOinZQIB8C2FxRAwwPDw9vwfsqr9ly9yV725D7I6XGw8PDu4jXbrizEKEtvmf3XnCe4OHh4cW8tu2Uhw7tVp4fSPmLwMPDwzvHm7W4Zktph7rau/Dw8PDuxpsV2ZuBqryMjlJqPDw8vA/xNgdG0h5rBwXqoSs8PDy8NW/W9Hp9SCRxbXtv/irx8PDwzvH2MUF7ALQLSn7/6lE0PDw8vD+vb2LHkpdgSFk9AAAAAElFTkSuQmCC'>
        
          <img src="https://cdn.jsdelivr.net/gh/xaoxuu/assets@19.1.9/logo/128/wechat.png">
        
        </a>
      
    
        <a class="-mob-share-weibo" title="微博" rel="external nofollow noopener noreferrer"
          
          href="http://service.weibo.com/share/share.php?url=https://www.itrhx.com/2019/08/23/A29-Python3-spider-C01/&title=Python3 爬虫学习笔记 C01 | TRHX'S BLOG&pics=https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.4/images/trhx.png&summary=
Python3 爬虫学习笔记第一章 ——【基本库 urllib 的使用】
"
          
          >
          
            <img src="https://cdn.jsdelivr.net/gh/xaoxuu/assets@19.1.9/logo/128/weibo.png">
          
        </a>
      
    
        <a class='qrcode' rel="external nofollow noopener noreferrer" href='data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAN4AAADeCAAAAAB3DOFrAAACu0lEQVR42u3aMW7DQAwEQP//00kbILG9S+piFaMqsCPrRsUdseTjEV9fP67fn/z+9tm9ye+8fuLjxIWHh4e3WHr7sNePTJb47JP86W9eMR4eHt4x3rPFJct6zbj2CMnXjIeHh3cfXnIYJJt+fmzk5TUeHh7ePXlJST0rwfPXioeHh3cHXlvO7j9pA4vjWQseHh5ezMu7SPf5+0h/Dw8PD2/dVZ8Fr20Bnce+9Wrx8PDwDvDyDXczAtUW6MkvFP+Jh4eHd4CXb/GzInhWLm8CCDw8PLxzvISRvIJ9i2t2VESHAR4eHt4xXh4EtAMHbSmcl+B1DIGHh4e35rXLaltTs/B3c8w8NicJHh4eXsBrI9R2y85jhbZwj+IJPDw8vAO8fC/dDxZshqvyQ+jNQYKHh4e35s2aTJuIod36V6MJeHh4eAd4szZY28TaxBbttxd0+fDw8PDSivSCpv4s1NgU022sjIeHh7fnFQFoCc5j37ZML/IVPDw8vGO8TWGdN8naptemAYaHh4f3Wd5mv80XlxTN7Rrw8PDwzvFmQ1TDsae4xXVBsY6Hh4d3gLc5EmatqdmwQhsTF8k0Hh4e3pqXbMGz9tjmiUnZHUUSeHh4eGte23Zqo4E85J1Fw2+OKzw8PLwDvPwAyLf+WSw7G8yKmnZ4eHh4h3l7Rl58J3FtO/61mhTDw8PDK3mzRtR+eGs2cFBEwHh4eHj/wkuOinZQIB8C2FxRAwwPDw9vwfsqr9ly9yV725D7I6XGw8PDu4jXbrizEKEtvmf3XnCe4OHh4cW8tu2Uhw7tVp4fSPmLwMPDwzvHm7W4Zktph7rau/Dw8PDuxpsV2ZuBqryMjlJqPDw8vA/xNgdG0h5rBwXqoSs8PDy8NW/W9Hp9SCRxbXtv/irx8PDwzvH2MUF7ALQLSn7/6lE0PDw8vD+vb2LHkpdgSFk9AAAAAElFTkSuQmCC'>
        
          <img src="https://cdn.jsdelivr.net/gh/xaoxuu/assets@19.1.9/logo/128/qrcode.png">
        
        </a>
      
    
  </div>
</div>


    </div>
  </section>


            <div class="prev-next">
                
                    <section class="prev">
                        <span class="art-item-left">
                            <h6><i class="fas fa-chevron-left" aria-hidden="true"></i>&nbsp;上一页</h6>
                            <h4>
                                <a href="/2019/08/23/A30-Python3-spider-C02/" rel="prev" title="Python3 爬虫学习笔记 C02">
                                  
                                      Python3 爬虫学习笔记 C02
                                  
                                </a>
                            </h4>
                            
                                
                                <h6 class="tags">
                                    <a class="tag" href="/tags/爬虫/"><i class="fas fa-tags fa-fw" aria-hidden="true"></i>&nbsp;爬虫</a> <a class="tag" href="/tags/requests/"><i class="fas fa-tags fa-fw" aria-hidden="true"></i>&nbsp;requests</a>
                                </h6>
                            
                        </span>
                    </section>
                
                
                    <section class="next">
                        <span class="art-item-right" aria-hidden="true">
                            <h6>下一页&nbsp;<i class="fas fa-chevron-right" aria-hidden="true"></i></h6>
                            <h4>
                                <a href="/2019/08/23/A24-instant.page/" rel="prev" title="一个 JS 脚本实现网站预加载，提升页面加载速度">
                                    
                                        一个 JS 脚本实现网站预加载，提升页面加载速度
                                    
                                </a>
                            </h4>
                            
                                
                                <h6 class="tags">
                                    <a class="tag" href="/tags/instant-page/"><i class="fas fa-tags fa-fw" aria-hidden="true"></i>&nbsp;instant.page</a> <a class="tag" href="/tags/JS-预加载/"><i class="fas fa-tags fa-fw" aria-hidden="true"></i>&nbsp;JS 预加载</a>
                                </h6>
                            
                        </span>
                    </section>
                
            </div>
        
      </section>
    </article>
  

    <!-- 显示推荐文章和评论 -->


  <article class="post white-box comments">
    <section class="article typo">
      <h4><i class="fas fa-comments fa-fw" aria-hidden="true"></i>&nbsp;评论</h4>
      
      
        <section id="comments">
          <div id="gitalk-container"></div>
        </section>
      
      
    </section>
  </article>


<!-- 根据页面mathjax变量决定是否加载MathJax数学公式js -->


  <script>
    window.subData = {
      title: 'Python3 爬虫学习笔记 C01',
      tools: true
    }
  </script>


</div>
<aside class='l_side'>
  
    
            <section class='widget author'>
  <div class='content pure'>
    
      <div class='avatar'>
        <img class='avatar' src='https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.6/images/trhx.png'/>
      </div>
    
    
      <div class='text'>
        
          <h2>๑۩ﺴ&nbsp;&nbsp;TRHX&nbsp;&nbsp;ﺴ۩๑</h2>
        
        
      </div>
    
    <!-- 彩色滚动字体 -->
    <div id="binft" align="center"></div>
    <script>
      var binft = function (r) {
        function t() {
          return b[Math.floor(Math.random() * b.length)]
        }  
        function e() {
          return String.fromCharCode(94 * Math.random() + 33)
        }
        function n(r) {
          for (var n = document.createDocumentFragment(), i = 0; r > i; i++) {
            var l = document.createElement("span");
            l.textContent = e(), l.style.color = t(), n.appendChild(l)
          }
          return n
        }
        function i() {
          var t = o[c.skillI];
          c.step ? c.step-- : (c.step = g, c.prefixP < l.length ? (c.prefixP >= 0 && (c.text += l[c.prefixP]), c.prefixP++) : "forward" === c.direction ? c.skillP < t.length ? (c.text += t[c.skillP], c.skillP++) : c.delay ? c.delay-- : (c.direction = "backward", c.delay = a) : c.skillP > 0 ? (c.text = c.text.slice(0, -1), c.skillP--) : (c.skillI = (c.skillI + 1) % o.length, c.direction = "forward")), r.textContent = c.text, r.appendChild(n(c.prefixP < l.length ? Math.min(s, s + c.prefixP) : Math.min(s, t.length - c.skillP))), setTimeout(i, d)
        }
        var l = "",
        o = ["Stay Hungry, Stay Foolish!"].map(function (r) {
        return r + ""
        }),
        a = 2,
        g = 1,
        s = 5,
        d = 75,
        b = ["rgb(110,64,170)", "rgb(150,61,179)", "rgb(191,60,175)", "rgb(228,65,157)", "rgb(254,75,131)", "rgb(255,94,99)", "rgb(255,120,71)", "rgb(251,150,51)", "rgb(226,183,47)", "rgb(198,214,60)", "rgb(175,240,91)", "rgb(127,246,88)", "rgb(82,246,103)", "rgb(48,239,130)", "rgb(29,223,163)", "rgb(26,199,194)", "rgb(35,171,216)", "rgb(54,140,225)", "rgb(76,110,219)", "rgb(96,84,200)"],
        c = {
          text: "",
          prefixP: -s,
          skillI: 0,
          skillP: 0,
          direction: "forward",
          delay: a,
          step: g
        };
        i()
        };
        binft(document.getElementById('binft'));
    </script>
    
      <div class="social-wrapper">
        
          
            <a href="https://github.com/TRHX"
              class="social fab fa-github flat-btn"
              target="_blank"
              rel="external nofollow noopener noreferrer">
            </a>
          
        
            <a href="https://itrhx.blog.csdn.net/"
              class="social fab fa-cuttlefish flat-btn"
              target="_blank"
              rel="external nofollow noopener noreferrer">
            </a>
          
        
            <a href="https://www.zhihu.com/people/tan-70-56/activities"
              class="social fab fa-zhihu flat-btn"
              target="_blank"
              rel="external nofollow noopener noreferrer">
            </a>
          
        
            <a href="mailto:admin@itrhx.com"
              class="social fas fa-envelope flat-btn"
              target="_blank"
              rel="external nofollow noopener noreferrer">
            </a>
          
        
            <a href="http://wpa.qq.com/msgrd?v=3&uin=2273902448&site=qq&menu=yes"
              class="social fab fa-qq flat-btn"
              target="_blank"
              rel="external nofollow noopener noreferrer">
            </a>
          
        
      </div>
    
  </div>
</section>

          
  <section class='widget toc-wrapper'>
    
<header class='pure'>
  <div><i class="fas fa-list fa-fw" aria-hidden="true"></i>&nbsp;&nbsp;本文目录</div>
  
    <div class='wrapper'><a class="s-toc rightBtn" rel="external nofollow noopener noreferrer" href="javascript:void(0)"><i class="fas fa-thumbtack fa-fw"></i></a></div>
  
</header>

    <div class='content pure'>
      <ol class="toc"><li class="toc-item toc-level-1"><a class="toc-link" href="#【1-1】-urllib-简介"><span class="toc-text"> 【1.1】 urllib 简介</span></a></li><li class="toc-item toc-level-1"><a class="toc-link" href="#【1-2】-urllib-request-发送请求"><span class="toc-text"> 【1.2】 urllib.request 发送请求</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#【1-2-1】-urllib-request-urlopen"><span class="toc-text">【1.2.1】 urllib.request.urlopen()</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#【1-2-1-1】-基本使用方法"><span class="toc-text">【1.2.1.1】 基本使用方法</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#【1-2-1-2】-添加参数"><span class="toc-text">【1.2.1.2】 添加参数</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#●-data-参数"><span class="toc-text">● data 参数</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#●-timeout-参数"><span class="toc-text">● timeout 参数</span></a></li></ol></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#【1-2-2】-urllib-request-Request"><span class="toc-text">【1.2.2】 urllib.request.Request()</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#【1-2-3】-urllib-request-urlretrieve"><span class="toc-text">【1.2.3】 urllib.request.urlretrieve()</span></a></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#【1-3】-urllib-error-异常处理"><span class="toc-text"> 【1.3】 urllib.error 异常处理</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#【1-3-1】-URLError"><span class="toc-text">【1.3.1】 URLError</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#【1-3-2】-HTTPError"><span class="toc-text">【1.3.2】 HTTPError</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#【1-3-3】-进阶用法"><span class="toc-text">【1.3.3】 进阶用法</span></a></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#【1-4】-urllib-parse-解析-URL"><span class="toc-text"> 【1.4】 urllib.parse 解析 URL</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#【1-4-1】-urllib-parse-urlencode"><span class="toc-text">【1.4.1】 urllib.parse.urlencode()</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#【1-4-2】-urllib-parse-parse-qs"><span class="toc-text">【1.4.2】 urllib.parse.parse_qs()</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#【1-4-3】-urllib-parse-parse-qsl"><span class="toc-text">【1.4.3】 urllib.parse.parse_qsl()</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#【1-4-4】-urllib-parse-urlparse"><span class="toc-text">【1.4.4】 urllib.parse.urlparse()</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#【1-4-5】-urllib-parse-urlunparse"><span class="toc-text">【1.4.5】 urllib.parse.urlunparse()</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#【1-4-6】-urllib-parse-urlsplit"><span class="toc-text">【1.4.6】 urllib.parse.urlsplit()</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#【1-4-7】-urllib-parse-urlunsplit"><span class="toc-text">【1.4.7】 urllib.parse.urlunsplit()</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#【1-4-8】-urllib-parse-urljoin"><span class="toc-text">【1.4.8】 urllib.parse.urljoin()</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#【1-4-9】-urllib-parse-quote"><span class="toc-text">【1.4.9】 urllib.parse.quote()</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#【1-4-10】-urllib-parse-unquote"><span class="toc-text">【1.4.10】 urllib.parse.unquote()</span></a></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#【1-5】-urllib-robotparser-爬取权限判断"><span class="toc-text"> 【1.5】 urllib.robotparser 爬取权限判断</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#【1-5-1】-Robots-协议简介"><span class="toc-text">【1.5.1】 Robots 协议简介</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#【1-5-2】-RobotFileParser-类常用方法"><span class="toc-text">【1.5.2】 RobotFileParser 类常用方法</span></a></li></ol></li></ol>
    </div>
  </section>


            <section class='widget grid'>
  
<header class='pure'>
  <div><i class="fas fa-map-signs fa-fw" aria-hidden="true"></i>&nbsp;&nbsp;站内导航</div>
  
</header>

  <div class='content pure'>
    <ul class="grid navgation">
      
        <li><a class="flat-box" title="/" href="/"
          
          
          id="home">
          
            <i class="fas fa-clock fa-fw" aria-hidden="true"></i>
          
          所有文章
        </a></li>
      
        <li><a class="flat-box" title="/archives/" href="/archives/"
          
          
          id="archives">
          
            <i class="fas fa-archive fa-fw" aria-hidden="true"></i>
          
          文章归档
        </a></li>
      
        <li><a class="flat-box" title="/atom.xml" href="/atom.xml"
          
          
          id="atom.xml">
          
            <i class="fas fa-rss fa-fw" aria-hidden="true"></i>
          
          RSS订阅
        </a></li>
      
        <li><a class="flat-box" title="/friends/" href="/friends/"
          
          
          id="friends">
          
            <i class="fas fa-link fa-fw" aria-hidden="true"></i>
          
          我的友链
        </a></li>
      
        <li><a class="flat-box" title="/comments/" href="/comments/"
          
          
            target="_blank"
          
          id="comments">
          
            <i class="fas fa-comments fa-fw" aria-hidden="true"></i>
          
          留言板
        </a></li>
      
        <li><a class="flat-box" title="/about/" href="/about/"
          
            rel="nofollow"
          
          
          id="about">
          
            <i class="fas fa-info-circle fa-fw" aria-hidden="true"></i>
          
          关于博主
        </a></li>
      
    </ul>
  </div>
</section>

          
            <section class='widget plain'>
  
<header class='pure'>
  <div><i class="fas fa-handshake fa-fw" aria-hidden="true"></i>&nbsp;&nbsp;欢迎光临</div>
  
</header>

  <div class='content pure'>
    <script src="https://pv.sohu.com/cityjson?ie=utf-8"></script><script>var getOSAndBrowser=function(){var os=navigator.platform;var userAgent=navigator.userAgent;var info="";var tempArray="";if(os.indexOf("Win")>-1){if(userAgent.indexOf("Windows NT 5.0")>-1){info+="Win2000"}else if(userAgent.indexOf("Windows NT 5.1")>-1){info+="WinXP"}else if(userAgent.indexOf("Windows NT 5.2")>-1){info+="Win2003"}else if(userAgent.indexOf("Windows NT 6.0")>-1){info+="WindowsVista"}else if(userAgent.indexOf("Windows NT 6.1")>-1||userAgent.indexOf("Windows 7")>-1){info+="Win7"}else if(userAgent.indexOf("Windows NT 6.2")>-1||userAgent.indexOf("Windows 8")>-1){info+="Win8"}else if(userAgent.indexOf("Windows NT 6.3")>-1||userAgent.indexOf("Windows 8.1")>-1){info+="Win8.1"}else if(userAgent.indexOf("Windows NT 10.0")>-1||userAgent.indexOf("Windows 10")>-1){info+="Win10"}else{info+="Other"}}else if(os.indexOf("Mac")>-1){info+="Mac"}else if(os.indexOf("X11")>-1){info+="Unix"}else if(os.indexOf("Linux")>-1){info+="Linux"}else{info+="Other"}info+="/";var isOpera=userAgent.indexOf("Opera")>-1;var isIE=userAgent.indexOf("compatible")>-1&&userAgent.indexOf("MSIE")>-1&&!isOpera;var isEdge=userAgent.toLowerCase().indexOf("edge")>-1&&!isIE;var isIE11=(userAgent.toLowerCase().indexOf("trident")>-1&&userAgent.indexOf("rv")>-1);if(/[Ff]irefox(\/\d+\.\d+)/.test(userAgent)){tempArray=/([Ff]irefox)\/(\d+\.\d+)/.exec(userAgent);info+=tempArray[1]+tempArray[2]}else if(isIE){var version="";var reIE=new RegExp("MSIE (\\d+\\.\\d+);");reIE.test(userAgent);var fIEVersion=parseFloat(RegExp["$1"]);if(fIEVersion==7){version="IE7"}else if(fIEVersion==8){version="IE8"}else if(fIEVersion==9){version="IE9"}else if(fIEVersion==10){version="IE10"}else{version="0"}info+=version}else if(isEdge){info+="Edge"}else if(isIE11){info+="IE11"}else if(/[Cc]hrome\/\d+/.test(userAgent)){tempArray=/([Cc]hrome)\/(\d+)/.exec(userAgent);info+=tempArray[1]+tempArray[2]}else if(/[Vv]ersion\/\d+\.\d+\.\d+(\.\d)* *[Ss]afari/.test(userAgent)){tempArray=/[Vv]ersion\/(\d+\.\d+\.\d+)(\.\d)* *([Ss]afari)/.exec(userAgent);info+=tempArray[3]+tempArray[1]}else if(/[Oo]pera.+[Vv]ersion\/\d+\.\d+/.test(userAgent)){tempArray=/([Oo]pera).+[Vv]ersion\/(\d+)\.\d+/.exec(userAgent);info+=tempArray[1]+tempArray[2]}else{info+="unknown"}return info};document.write("欢迎来自"+returnCitySN["cname"]+"的朋友");document.write("<br>您的 IP 是："+returnCitySN["cip"]);document.write("<br>您使用的是："+getOSAndBrowser());</script>
  </div>
</section>

          
            <section class='widget plain'>
  
<header class='pure'>
  <div><i class="fas fa-map-marked-alt fa-fw" aria-hidden="true"></i>&nbsp;&nbsp;访客地图</div>
  
    <a class="rightBtn"
    
      rel="external nofollow noopener noreferrer"
    
    
      target="_blank"
    
    href="https://www.revolvermaps.com/"
    title="https://www.revolvermaps.com/">
    <i class="fas fa-info-circle fa-fw"></i></a>
  
</header>

  <div class='content pure'>
    <div style="position: relative;"><script type="text/javascript" src="//rf.revolvermaps.com/0/0/4.js?i=5eyl60h41k1&amp;m=6&amp;h=122&amp;c=ff0000&amp;r=30" async="async"></script></div>
  </div>
</section>

          
            <section class='widget plain'>
  
<header class='pure'>
  <div><i class="fas fa-comments fa-fw" aria-hidden="true"></i>&nbsp;&nbsp;交流群组</div>
  
</header>

  <div class='content pure'>
    <p><img src="https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.6/images/groups.png"><a id="goroups_button" class="goroups_button_new" target="_blank" href="https://jq.qq.com/?_wv=1027&k=5F6HRuG">QQ 交流群</a><a id="goroups_button" class="goroups_button_new" target="_blank" href="https://t.me/joinchat/Mro2h0uVt0DpDvOofWG65A">Telegram 交流群</a></p>

  </div>
</section>

          
  <section class='widget category'>
    
<header class='pure'>
  <div><i class="fas fa-folder-open fa-fw" aria-hidden="true"></i>&nbsp;&nbsp;文章分类</div>
  
    <a class="rightBtn"
    
      rel="nofollow"
    
    
    href="/categories/"
    title="categories/">
    <i class="fas fa-expand-arrows-alt fa-fw"></i></a>
  
</header>

    <div class='content pure'>
      <ul class="entry">
        
          <li><a class="flat-box" title="/categories/BLOG/" href="/categories/BLOG/"><div class='name'>BLOG</div><div class='badge'>(4)</div></a></li>
        
          <li><a class="flat-box" title="/categories/CDN/" href="/categories/CDN/"><div class='name'>CDN</div><div class='badge'>(1)</div></a></li>
        
          <li><a class="flat-box" title="/categories/Hexo/" href="/categories/Hexo/"><div class='name'>Hexo</div><div class='badge'>(9)</div></a></li>
        
          <li><a class="flat-box" title="/categories/Java/" href="/categories/Java/"><div class='name'>Java</div><div class='badge'>(1)</div></a></li>
        
          <li><a class="flat-box" title="/categories/Linux/" href="/categories/Linux/"><div class='name'>Linux</div><div class='badge'>(2)</div></a></li>
        
          <li><a class="flat-box" title="/categories/Markdown/" href="/categories/Markdown/"><div class='name'>Markdown</div><div class='badge'>(2)</div></a></li>
        
          <li><a class="flat-box" title="/categories/Python3-学习笔记/" href="/categories/Python3-学习笔记/"><div class='name'>Python3 学习笔记</div><div class='badge'>(39)</div></a></li>
        
          <li><a class="flat-box child" title="/categories/Python3-学习笔记/基础学习/" href="/categories/Python3-学习笔记/基础学习/"><div class='name'>基础学习</div><div class='badge'>(9)</div></a></li>
        
          <li><a class="flat-box child" title="/categories/Python3-学习笔记/学习经验/" href="/categories/Python3-学习笔记/学习经验/"><div class='name'>学习经验</div><div class='badge'>(4)</div></a></li>
        
          <li><a class="flat-box child" title="/categories/Python3-学习笔记/爬虫学习/" href="/categories/Python3-学习笔记/爬虫学习/"><div class='name'>爬虫学习</div><div class='badge'>(18)</div></a></li>
        
          <li><a class="flat-box child" title="/categories/Python3-学习笔记/爬虫实战/" href="/categories/Python3-学习笔记/爬虫实战/"><div class='name'>爬虫实战</div><div class='badge'>(8)</div></a></li>
        
          <li><a class="flat-box" title="/categories/VPS/" href="/categories/VPS/"><div class='name'>VPS</div><div class='badge'>(1)</div></a></li>
        
          <li><a class="flat-box" title="/categories/WEB前端/" href="/categories/WEB前端/"><div class='name'>WEB前端</div><div class='badge'>(2)</div></a></li>
        
          <li><a class="flat-box" title="/categories/图床/" href="/categories/图床/"><div class='name'>图床</div><div class='badge'>(1)</div></a></li>
        
      </ul>
    </div>
  </section>


  <section class='widget tagcloud'>
    
<header class='pure'>
  <div><i class="fas fa-fire fa-fw" aria-hidden="true"></i>&nbsp;&nbsp;热门标签</div>
  
    <a class="rightBtn"
    
      rel="nofollow"
    
    
    href="/tags/"
    title="tags/">
    <i class="fas fa-expand-arrows-alt fa-fw"></i></a>
  
</header>

    <div class='content pure'>
      <a href="/tags/12306/" style="font-size: 14px; color: #999">12306</a> <a href="/tags/58同城/" style="font-size: 14px; color: #999">58同城</a> <a href="/tags/Ajax/" style="font-size: 14px; color: #999">Ajax</a> <a href="/tags/BLOG/" style="font-size: 16.5px; color: #888">BLOG</a> <a href="/tags/Beautiful-Soup/" style="font-size: 14px; color: #999">Beautiful Soup</a> <a href="/tags/CDN/" style="font-size: 14px; color: #999">CDN</a> <a href="/tags/Coding-Pages/" style="font-size: 14px; color: #999">Coding Pages</a> <a href="/tags/Deepin/" style="font-size: 14px; color: #999">Deepin</a> <a href="/tags/Elicpse/" style="font-size: 14px; color: #999">Elicpse</a> <a href="/tags/GitHub-Pages/" style="font-size: 14px; color: #999">GitHub Pages</a> <a href="/tags/Github/" style="font-size: 14px; color: #999">Github</a> <a href="/tags/Github-Pages/" style="font-size: 14px; color: #999">Github Pages</a> <a href="/tags/HTTPS/" style="font-size: 16.5px; color: #888">HTTPS</a> <a href="/tags/Hexo/" style="font-size: 21.5px; color: #666">Hexo</a> <a href="/tags/ICP备案/" style="font-size: 14px; color: #999">ICP备案</a> <a href="/tags/JDBC/" style="font-size: 14px; color: #999">JDBC</a> <a href="/tags/JS-预加载/" style="font-size: 14px; color: #999">JS 预加载</a> <a href="/tags/Markdown/" style="font-size: 16.5px; color: #888">Markdown</a> <a href="/tags/Material-X/" style="font-size: 14px; color: #999">Material X</a> <a href="/tags/MongoDB/" style="font-size: 14px; color: #999">MongoDB</a> <a href="/tags/MySQL/" style="font-size: 14px; color: #999">MySQL</a> <a href="/tags/PEP8/" style="font-size: 14px; color: #999">PEP8</a> <a href="/tags/PicGo/" style="font-size: 14px; color: #999">PicGo</a> <a href="/tags/Pygame/" style="font-size: 14px; color: #999">Pygame</a> <a href="/tags/Python/" style="font-size: 16.5px; color: #888">Python</a> <a href="/tags/Redis/" style="font-size: 14px; color: #999">Redis</a> <a href="/tags/SEO/" style="font-size: 14px; color: #999">SEO</a> <a href="/tags/SQL-Server-2012/" style="font-size: 14px; color: #999">SQL Server 2012</a> <a href="/tags/SSR/" style="font-size: 14px; color: #999">SSR</a> <a href="/tags/Selenium/" style="font-size: 19px; color: #777">Selenium</a> <a href="/tags/Ubuntu/" style="font-size: 14px; color: #999">Ubuntu</a> <a href="/tags/User-Agent/" style="font-size: 14px; color: #999">User-Agent</a> <a href="/tags/VMware/" style="font-size: 16.5px; color: #888">VMware</a> <a href="/tags/VPS/" style="font-size: 14px; color: #999">VPS</a> <a href="/tags/XPath/" style="font-size: 14px; color: #999">XPath</a> <a href="/tags/if语句/" style="font-size: 14px; color: #999">if语句</a> <a href="/tags/input-函数/" style="font-size: 14px; color: #999">input()函数</a> <a href="/tags/instant-page/" style="font-size: 14px; color: #999">instant.page</a> <a href="/tags/jsDelivr/" style="font-size: 16.5px; color: #888">jsDelivr</a> <a href="/tags/lxml/" style="font-size: 14px; color: #999">lxml</a> <a href="/tags/pyspider/" style="font-size: 16.5px; color: #888">pyspider</a> <a href="/tags/requests/" style="font-size: 14px; color: #999">requests</a> <a href="/tags/spfk/" style="font-size: 14px; color: #999">spfk</a> <a href="/tags/urllib/" style="font-size: 14px; color: #999">urllib</a> <a href="/tags/while循环/" style="font-size: 14px; color: #999">while循环</a> <a href="/tags/主题个性化/" style="font-size: 14px; color: #999">主题个性化</a> <a href="/tags/代理/" style="font-size: 14px; color: #999">代理</a> <a href="/tags/公安备案/" style="font-size: 14px; color: #999">公安备案</a> <a href="/tags/函数/" style="font-size: 14px; color: #999">函数</a> <a href="/tags/列表/" style="font-size: 14px; color: #999">列表</a> <a href="/tags/变量/" style="font-size: 14px; color: #999">变量</a> <a href="/tags/哔哩哔哩/" style="font-size: 14px; color: #999">哔哩哔哩</a> <a href="/tags/图床/" style="font-size: 14px; color: #999">图床</a> <a href="/tags/图形验证码/" style="font-size: 14px; color: #999">图形验证码</a> <a href="/tags/垃圾/" style="font-size: 14px; color: #999">垃圾</a> <a href="/tags/备份/" style="font-size: 14px; color: #999">备份</a> <a href="/tags/字典/" style="font-size: 14px; color: #999">字典</a> <a href="/tags/安居客/" style="font-size: 14px; color: #999">安居客</a> <a href="/tags/年终总结/" style="font-size: 14px; color: #999">年终总结</a> <a href="/tags/异常/" style="font-size: 14px; color: #999">异常</a> <a href="/tags/技巧/" style="font-size: 14px; color: #999">技巧</a> <a href="/tags/操作列表/" style="font-size: 14px; color: #999">操作列表</a> <a href="/tags/数据类型/" style="font-size: 14px; color: #999">数据类型</a> <a href="/tags/文件/" style="font-size: 14px; color: #999">文件</a> <a href="/tags/文件储存/" style="font-size: 14px; color: #999">文件储存</a> <a href="/tags/无界面浏览器/" style="font-size: 14px; color: #999">无界面浏览器</a> <a href="/tags/模块/" style="font-size: 14px; color: #999">模块</a> <a href="/tags/正则表达式/" style="font-size: 14px; color: #999">正则表达式</a> <a href="/tags/滑动验证码/" style="font-size: 14px; color: #999">滑动验证码</a> <a href="/tags/点触验证码/" style="font-size: 14px; color: #999">点触验证码</a> <a href="/tags/爬虫/" style="font-size: 24px; color: #555">爬虫</a> <a href="/tags/猫眼电影/" style="font-size: 14px; color: #999">猫眼电影</a> <a href="/tags/瓜子二手车/" style="font-size: 14px; color: #999">瓜子二手车</a> <a href="/tags/类/" style="font-size: 14px; color: #999">类</a> <a href="/tags/继承/" style="font-size: 14px; color: #999">继承</a> <a href="/tags/编辑器/" style="font-size: 14px; color: #999">编辑器</a> <a href="/tags/虎扑论坛/" style="font-size: 14px; color: #999">虎扑论坛</a> <a href="/tags/豆瓣电影/" style="font-size: 14px; color: #999">豆瓣电影</a>
    </div>
  </section>

          
  <section class='widget music'>
    
<header class='pure'>
  <div><i class="fas fa-compact-disc fa-fw" aria-hidden="true"></i>&nbsp;&nbsp;最近在听</div>
  
    <a class="rightBtn"
    
      rel="external nofollow noopener noreferrer"
    
    
      target="_blank"
    
    href="https://music.163.com/#/user/home?id=1379654769"
    title="https://music.163.com/#/user/home?id=1379654769">
    <i class="far fa-heart fa-fw"></i></a>
  
</header>

    <div class='content pure'>
      
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/aplayer@1.7.0/dist/APlayer.min.css">
  <div class="aplayer"
    data-theme="#1BCDFC"
    
    
    data-mode="circulation"
    data-server="netease"
    data-type="playlist"
    data-id="3019271605"
    data-volume="0.7">
  </div>
  <script src="https://cdn.jsdelivr.net/npm/aplayer@1.7.0/dist/APlayer.min.js"></script>
  <script src="https://cdn.jsdelivr.net/npm/meting@1.1.0/dist/Meting.min.js"></script>


    </div>
  </section>


</aside>

<footer id="footer" class="clearfix">
    
    
      <div class="social-wrapper">
        
          
            <a href="https://github.com/TRHX"
              class="social fab fa-github flat-btn"
              target="_blank"
              rel="external nofollow noopener noreferrer">
            </a>
          
        
            <a href="https://itrhx.blog.csdn.net/"
              class="social fab fa-cuttlefish flat-btn"
              target="_blank"
              rel="external nofollow noopener noreferrer">
            </a>
          
        
            <a href="https://www.zhihu.com/people/tan-70-56/activities"
              class="social fab fa-zhihu flat-btn"
              target="_blank"
              rel="external nofollow noopener noreferrer">
            </a>
          
        
            <a href="mailto:admin@itrhx.com"
              class="social fas fa-envelope flat-btn"
              target="_blank"
              rel="external nofollow noopener noreferrer">
            </a>
          
        
            <a href="http://wpa.qq.com/msgrd?v=3&amp;uin=2273902448&amp;site=qq&amp;menu=yes"
              class="social fab fa-qq flat-btn"
              target="_blank"
              rel="external nofollow noopener noreferrer">
            </a>
          
        
      </div>
    
    <br>
    Copyright <i class="far fa-copyright"></i> 2018-2020
    <a href="https://www.itrhx.com/" target="_blank"> TRHX&#39;S BLOG </a>丨
    <img src="https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.6/images/icp.png">
    <a href="http://www.beian.miit.gov.cn/" target="_blank">鄂ICP备19003281号-4</a> 丨
    <!-- 网站运行时间 -->
    <span id="timeDate">正在载入...</span><span id="times"><span><i class="fa fa-spinner fa-spin"> </i></span></span>
    <script>
    var now = new Date(); 
    function createtime() {
      var grt= new Date("08/10/2018 17:38:00");//建站时间2018-08-10
      now.setTime(now.getTime()+250); 
      days = (now - grt ) / 1000 / 60 / 60 / 24; dnum = Math.floor(days); 
      hours = (now - grt ) / 1000 / 60 / 60 - (24 * dnum); hnum = Math.floor(hours); 
      if(String(hnum).length ==1 ){hnum = "0" + hnum;} minutes = (now - grt ) / 1000 /60 - (24 * 60 * dnum) - (60 * hnum); 
      mnum = Math.floor(minutes); if(String(mnum).length ==1 ){mnum = "0" + mnum;} 
      seconds = (now - grt ) / 1000 - (24 * 60 * 60 * dnum) - (60 * 60 * hnum) - (60 * mnum); 
      snum = Math.round(seconds); if(String(snum).length ==1 ){snum = "0" + snum;} 
      document.getElementById("timeDate").innerHTML = "本站已在风雨中度过了 "
      document.getElementById("times").innerHTML = dnum +" 天 " + hnum + " 小时 " + mnum + " 分 " + snum + " 秒"; 
      /*document.getElementById("times").style.color="#0056FF";*/
      }
      setInterval("createtime()",250);
    </script>丨
    <a href="/sitemap.xml" target="_blank">站点地图</a>丨
    <a href="https://tongji.baidu.com/web/welcome/ico?s=df0bc7c6bdbd80356ba4db429724ccad" target="_blank">站长统计</a><br><br>
    <div class="github-badge">
      <a style="color: #fff" rel="license" href="https://hexo.io/" target="_blank" title="由 Hexo 强力驱动">
      <span class="badge-subject">Powered</span><span class="badge-value bg-blue">Hexo</span></a>
    </div>
    <div class="github-badge">
      <a style="color: #fff" rel="license" href="https://github.com/" target="_blank" title="静态网页托管于 GitHub Pages 和 Coding Pages">
      <span class="badge-subject">Hosted</span><span class="badge-value bg-brightgreen">GitHub & Coding</span></a>
    </div>
    <div class="github-badge">
      <a style="color: #fff" rel="license" href="https://www.aliyun.com/" target="_blank" title="阿里云提供域名相关服务">
      <span class="badge-subject">DNR</span><span class="badge-value bg-blueviolet">Aliyun</span></a>
    </div>
    <div class="github-badge">
      <a style="color: #fff" rel="license" href="https://www.jsdelivr.com/" target="_blank" title="jsDelivr 提供 CDN 加速服务">
      <span class="badge-subject">CDN</span><span class="badge-value bg-orange">jsDelivr</span></a>
    </div>
    <div class="github-badge">
        <a style="color: #fff" rel="license" href="https://xaoxuu.com/wiki/material-x/" target="_blank" title="站点使用 Material X 主题">
      <span class="badge-subject">Theme</span><span class="badge-value bg-blue">Material X</span></a>
    </div>
    <div class="github-badge">
      <a style="color: #fff" rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/" target="_blank" title="本站点采用知识共享署名-非商业性使用-相同方式共享 4.0 国际许可协议进行许可">
      <span class="badge-subject"><i class="fa fa-copyright"></i></span><span class="badge-value bg-lightgrey">BY-NC-SA 4.0</span></a>
    </div>
    <div class="github-badge">
      <a style="color: #fff" rel="license" href="https://996.icu/" target="_blank" title="支持 996.ICU">
      <span class="badge-subject">Link</span><span class="badge-value bg-red">996.ICU</span></a>
    </div>
    <div class="github-badge">
      <span class="badge-subject">UV</span><span class="badge-value bg-orange" id="busuanzi_value_site_uv"></span>
    </div>
    <div class="github-badge">
      <span class="badge-subject">PV</span><span class="badge-value bg-brightgreen" id="busuanzi_value_site_pv"></span>
    </div>
    <div class="github-badge">
      <span class="badge-subject">WordCount</span><span class="badge-value bg-blueviolet">130.4k</span>
    </div>
</footer>


      <script>setLoadingBarProgress(60);</script>
    </div>
    <a class="s-top fas fa-arrow-up fa-fw" href='javascript:void(0)'></a>
  </div>
  <script src="https://cdn.jsdelivr.net/npm/jquery@3.3.1/dist/jquery.min.js"></script>

  <script>
    var GOOGLE_CUSTOM_SEARCH_API_KEY = "";
    var GOOGLE_CUSTOM_SEARCH_ENGINE_ID = "";
    var ALGOLIA_API_KEY = "";
    var ALGOLIA_APP_ID = "";
    var ALGOLIA_INDEX_NAME = "";
    var AZURE_SERVICE_NAME = "";
    var AZURE_INDEX_NAME = "";
    var AZURE_QUERY_KEY = "";
    var BAIDU_API_ID = "";
    var SEARCH_SERVICE = "hexo" || "hexo";
    var ROOT = "/"||"/";
    if(!ROOT.endsWith('/'))ROOT += '/';
  </script>

<script src="//instant.page/1.2.2" type="module" integrity="sha384-2xV8M5griQmzyiY3CDqh1dn4z3llDVqZDqzjzcY+jCBCk/a5fXJmuZ/40JJAPeoU"></script>


  <script async src="https://cdn.jsdelivr.net/npm/scrollreveal@4.0.5/dist/scrollreveal.min.js"></script>
  <script type="text/javascript">
    $(function() {
      const $reveal = $('.reveal');
      if ($reveal.length === 0) return;
      const sr = ScrollReveal({ distance: 0 });
      sr.reveal('.reveal');
    });
  </script>


  <script src="https://cdn.jsdelivr.net/npm/node-waves@0.7.6/dist/waves.min.js"></script>
  <script type="text/javascript">
    $(function() {
      Waves.attach('.flat-btn', ['waves-button']);
      Waves.attach('.float-btn', ['waves-button', 'waves-float']);
      Waves.attach('.float-btn-light', ['waves-button', 'waves-float', 'waves-light']);
      Waves.attach('.flat-box', ['waves-block']);
      Waves.attach('.float-box', ['waves-block', 'waves-float']);
      Waves.attach('.waves-image');
      Waves.init();
    });
  </script>


  <script async src="https://cdn.jsdelivr.net/gh/xaoxuu/cdn-busuanzi@2.3/js/busuanzi.pure.mini.js"></script>


    <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery-backstretch/2.0.4/jquery.backstretch.min.js"></script>
    <script type="text/javascript">
      $(function(){
        if ('.cover') {
          $('.cover').backstretch(
          ["https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.6/images/background/021.webp"],
          {
            duration: "5000",
            fade: "1500"
          });
        } else {
          $.backstretch(
          ["https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.6/images/background/021.webp"],
          {
            duration: "5000",
            fade: "1500"
          });
        }
      });
    </script>
  

  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/gitalk@1/dist/gitalk.css">
  <script src="https://cdn.jsdelivr.net/npm/gitalk@1/dist/gitalk.min.js"></script>
  <script src="https://cdn.bootcss.com/blueimp-md5/2.10.0/js/md5.min.js"></script>
  <script type="text/javascript">
    var gitalk = new Gitalk({
      clientID: "d19a84b9d9a2ddb2c6b9",
      clientSecret: "cec9feae5129a6106edc68ce06d167be8eb06021",
      repo: "trhx.github.io",
      owner: "TRHX",
      admin: "TRHX",
      
        id: md5(location.pathname),      // Ensure uniqueness and length less than 50
      
      distractionFreeMode: false  // Facebook-like distraction free mode
    });
    gitalk.render('gitalk-container');
  </script>


  <script src="https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.6/js/app.js"></script>


  <script src="https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.6/js/search.js"></script>


<!-- 复制 -->
<script src="https://cdn.jsdelivr.net/npm/clipboard@2/dist/clipboard.min.js"></script>
<script>
  let COPY_SUCCESS = "复制成功";
  let COPY_FAILURE = "复制失败";
  /*页面载入完成后，创建复制按钮*/
  !function (e, t, a) {
    /* code */
    var initCopyCode = function(){
      var copyHtml = '';
      copyHtml += '<button class="btn-copy" data-clipboard-snippet="">';
      copyHtml += '  <i class="fa fa-copy"></i><span>复制</span>';
      copyHtml += '</button>';
      $(".highlight .code pre").before(copyHtml);
      var clipboard = new ClipboardJS('.btn-copy', {
        target: function(trigger) {
          return trigger.nextElementSibling;
        }
      });

      clipboard.on('success', function(e) {
        //您可以加入成功提示
        console.info('Action:', e.action);
        console.info('Text:', e.text);
        console.info('Trigger:', e.trigger);
        success_prompt(COPY_SUCCESS);
        e.clearSelection();
      });
      clipboard.on('error', function(e) {
        //您可以加入失败提示
        console.error('Action:', e.action);
        console.error('Trigger:', e.trigger);
        fail_prompt(COPY_FAILURE);
      });
    }
    initCopyCode();

  }(window, document);

  /**
   * 弹出式提示框，默认1.5秒自动消失
   * @param message 提示信息
   * @param style 提示样式，有alert-success、alert-danger、alert-warning、alert-info
   * @param time 消失时间
   */
  var prompt = function (message, style, time)
  {
      style = (style === undefined) ? 'alert-success' : style;
      time = (time === undefined) ? 1500 : time*1000;
      $('<div>')
          .appendTo('body')
          .addClass('alert ' + style)
          .html(message)
          .show()
          .delay(time)
          .fadeOut();
  };

  // 成功提示
  var success_prompt = function(message, time)
  {
      prompt(message, 'alert-success', time);
  };

  // 失败提示
  var fail_prompt = function(message, time)
  {
      prompt(message, 'alert-danger', time);
  };

  // 提醒
  var warning_prompt = function(message, time)
  {
      prompt(message, 'alert-warning', time);
  };

  // 信息提示
  var info_prompt = function(message, time)
  {
      prompt(message, 'alert-info', time);
  };

</script>


<!-- fancybox -->
<script src="https://cdn.jsdelivr.net/gh/fancyapps/fancybox@3.5.7/dist/jquery.fancybox.min.js"></script>
<script>
  let LAZY_LOAD_IMAGE = "";
  $(".article-entry").find("fancybox").find("img").each(function () {
      var element = document.createElement("a");
      $(element).attr("data-fancybox", "gallery");
      $(element).attr("href", $(this).attr("src"));
      /* 图片采用懒加载处理时,
       * 一般图片标签内会有个属性名来存放图片的真实地址，比如 data-original,
       * 那么此处将原本的属性名src替换为对应属性名data-original,
       * 修改如下
       */
       if (LAZY_LOAD_IMAGE) {
         $(element).attr("href", $(this).attr("data-original"));
       }
      $(this).wrap(element);
  });
</script>


  <script>setLoadingBarProgress(100);</script>
  <!-- 十年之约 -->
  <!--<a target="_blank" href="http://join.foreverblog.cn" style="position:fixed;z-index:999;right:-5px;bottom: 25px;display: inline-block;width: 35px;border-radius: 5px;color:white;font-size:14px;line-height:17px;background: #1BC3FB;box-shadow: 0 0 5px #666;word-wrap: break-word;padding: 10px 6px;border: 2px solid white;">十年之约</a>-->
  <!-- 单击显示文字 -->
  <script type="text/javascript" src="https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.6/js/ClickShowText.js"></script>
  <!-- 点击冒点 -->
  <canvas width="1777" height="841" style="position: fixed; left: 0px; top: 0px; z-index: 2147483647; pointer-events: none;"></canvas><script src="https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.6/js/maodian.js"></script>
  <!-- 浏览器搞笑标题 -->
  <script type="text/javascript" src="https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.6/js/FunnyTitle.js"></script>
  <!-- 样式一（鼠标点击更换样式） -->
  <!--<script src="https://g.joyinshare.com/hc/ribbon.min.js" type="text/javascript"></script>-->
  <!-- 样式二（飘动的彩带） -->
  <!--<script src="https://g.joyinshare.com/hc/piao.js" type="text/javascript"></script>-->
  <!-- 数字雨 -->
  <!--<canvas id="canvas" width="1920" height="1080"></canvas>
  <script type="text/javascript" src="https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.6/js/DigitalRain.js"></script>-->
  <!--动态线条背景-->
  <!--<script type="text/javascript" color="27,195,251" opacity='0.7' zIndex="-5" count="200" src="//cdn.bootcss.com/canvas-nest.js/1.0.0/canvas-nest.min.js"></script>-->
  <!-- 速度优化脚本 -->
  <script src="https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.6/js/instantclick-1.2.2.js" type="module"></script>
  <!-- 悟空 -->
  <!--<div id="wukong"><marquee direction="right"><img src="https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.6/images/wukong.gif"></marquee></div>-->
  <!-- 雪花 -->
  <script src="https://cdn.jsdelivr.net/gh/TRHX/CDN-for-itrhx.com@3.0.6/js/snow3.js"></script>
</body>
</html>