Python PycURL 网络编程实例及自动处理Cookie详解

Python PycURL 网络编程实例

在使用urllib的时候经常会死掉，以前debug过，是没有设置 timing out 所以超时后就会死掉。
PycURL是curl的python库，虽然有些curl的功能没有实现，但是还是很强劲的。

curl是非常强劲的一个工具，
google内部用它来 debug GDATA API. Using cURL to interact with Google data services

可以去 http://pycurl.s*o*urce*forge.net/ 下载最新的PycURL。

简单的PycURL例子

import pycurl
import StringIO
 
url = "http://www.g***oogle.com/"
crl = pycurl.Curl()
crl.setopt(pycurl.VERBOSE,1)
crl.setopt(pycurl.FOLLOWLOCATION, 1)
crl.setopt(pycurl.MAXREDIRS, 5)
crl.fp = StringIO.StringIO()
crl.setopt(pycurl.URL, url)
crl.setopt(crl.WRITEFUNCTION, crl.fp.write)
crl.perform()
print crl.fp.getvalue()

PycURL 自动处理cookie

import pycurl
import StringIO
 
url = "http://www.g***oogle.com/"
crl = pycurl.Curl()
crl.setopt(pycurl.VERBOSE,1)
crl.setopt(pycurl.FOLLOWLOCATION, 1)
crl.setopt(pycurl.MAXREDIRS, 5)
crl.fp = StringIO.StringIO()
crl.setopt(pycurl.URL, url)
crl.setopt(crl.WRITEFUNCTION, crl.fp.write)
 
# Option -b/--cookie  Cookie string or file to read cookies from
# Note: must be a string, not a file object.
crl.setopt(pycurl.COOKIEFILE, "cookie_file_name")
 
# Option -c/--cookie-jar  Write cookies to this file after operation
# Note: must be a string, not a file object.
crl.setopt(pycurl.COOKIEJAR, "cookie_file_name")
 
crl.perform()
print crl.fp.getvalue()

PycURL 实现POST方法

import pycurl
import StringIO
import urllib
 
url = "http://www.g***oogle.com/"
post_data_dic = {"name":"value"}
crl = pycurl.Curl()
crl.setopt(pycurl.VERBOSE,1)
crl.setopt(pycurl.FOLLOWLOCATION, 1)
crl.setopt(pycurl.MAXREDIRS, 5)
#crl.setopt(pycurl.AUTOREFERER,1)
 
crl.setopt(pycurl.CONNECTTIMEOUT, 60)
crl.setopt(pycurl.TIMEOUT, 300)
#crl.setopt(pycurl.PROXY,proxy)
crl.setopt(pycurl.HTTPPROXYTUNNEL,1)
#crl.setopt(pycurl.NOSIGNAL, 1)
crl.fp = StringIO.StringIO()
crl.setopt(pycurl.USERAGENT, "dhgu hoho")
 
# Option -d/--data    HTTP POST data
crl.setopt(crl.POSTFIELDS,  urllib.urlencode(post_data_dic))
 
crl.setopt(pycurl.URL, url)
crl.setopt(crl.WRITEFUNCTION, crl.fp.write)
crl.perform()
 
print crl.fp.getvalue()

urllib 超时设置

import socket
socket.setdefaulttimeout(5.0)

Python中Cookie的处理：自动处理Cookie，保存为Cookie文件，从文件载入Cookie

Python中的Cookie相关的库

Python中有自带的模块：cookielib

可以处理cookie相关的很多事情。

包括

自动处理cookie：cookie数据是保存在内存中
自动处理cookie：支持Cookie数据存放到文件中
其中有两种格式：
LWP
Mozilla
进一步，再支持：
将cookie存到文件里面
从文件中载入cookie

更多详细格式和解释，请直接看下面的代码。


Python中自动处理Cookie，将Cookie保存到文件，从文件中读取Cookie

Python代码：

#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Version:    2013-01-15
Author:     Crifan
Contact:    admin (at) crifan.com
"""
 
import os;
import cookielib;
import urllib2;
 
def pythonAutoHandleCookie():
    """
        Demo how to auto handle cookie in Python
            cookies in memory
            cookies in file:
                save cookie to file
                    LWP     format
                    Mozilla format
                load cookie from file
    """
 
    print "1. Demo how to auto handle cookie (in memory)";
    cookieJarInMemory = cookielib.CookieJar();
    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookieJarInMemory));
    urllib2.install_opener(opener);
    print "after init, cookieJarInMemory=",cookieJarInMemory; #after init, cookieJarInMemory=     #!!! following urllib2 will auto handle cookies
    demoUrl = "http://www.g***oogle.com/";
    response = urllib2.urlopen(demoUrl);
    #here, we already got response cookies
    print "after urllib2.urlopen, cookieJarInMemory=",cookieJarInMemory;
    #after urllib2.urlopen, cookieJarInMemory= 
     
     
    print "2. Demo how to auto handle cookie in file, LWP format";
    cookieFilenameLWP = "localCookiesLWP.txt";
    cookieJarFileLWP = cookielib.LWPCookieJar(cookieFilenameLWP);
    #will create (and save to) new cookie file
    cookieJarFileLWP.save();
    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookieJarFileLWP));
    urllib2.install_opener(opener);
    #!!! following urllib2 will auto handle cookies
    demoUrl = "http://www.g***oogle.com/";
    response = urllib2.urlopen(demoUrl);
    #update cookies, save cookies into file
    cookieJarFileLWP.save();
    #for demo, print cookies in file
    print "LWP cookies:";
    print open(cookieFilenameLWP).read(os.path.getsize(cookieFilenameLWP));
    # #LWP-Cookies-2.0
    # Set-Cookie3: PREF="ID=34c1415b570a93ae:FF=0:NW=1:TM=1358236121:LM=1358236121:S=gEVVojW4x37ht5n-"; path="/"; domain=".google.com"; path_spec; domain_dot; expires="2015-01-15 07:48:41Z"; version=0
    # Set-Cookie3: NID="67=JI_uEwUm5GDrQ_vCwAp2z_YGU7MdLm5CLMa4CNLF7RQuTDMzrrk1EjRddGcnpoFbht81LaV9spxZQQInf0mPS6lDrvcRqBBL5NOTmy8SwOzA6HWC3iTIo4-o3fO1Udkv"; path="/"; domain=".google.com.hk"; path_spec; domain_dot; expires="2013-07-17 07:48:41Z"; HttpOnly=None; version=0
    # Set-Cookie3: PREF="ID=8f7e4efca89bdb1b:U=f85a4afa4db021aa:FF=2:LD=zh-CN:NW=1:TM=1358236121:LM=1358236121:S=2WR59hDWutdnUJtF"; path="/"; domain=".google.com.hk"; path_spec; domain_dot; expires="2015-01-15 07:48:41Z"; version=0
 
    print "3. Demo how to auto handle cookie in file, Mozilla Format";
    cookieFilenameMozilla = "localCookiesMozilla.txt";
    cookieJarFileMozilla = cookielib.MozillaCookieJar(cookieFilenameMozilla);
    #will create (and save to) new cookie file
    cookieJarFileMozilla.save();
    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookieJarFileMozilla));
    urllib2.install_opener(opener);
    #!!! following urllib2 will auto handle cookies
    demoUrl = "http://www.g***oogle.com/";
    response = urllib2.urlopen(demoUrl);
    #update cookies, save cookies into file
    cookieJarFileMozilla.save();
    #for demo, print cookies in file
    print "Mozilla cookies:";
    print open(cookieFilenameMozilla).read(os.path.getsize(cookieFilenameMozilla));
    # # Netscape HTTP Cookie File
    # # http://www.*ne*tsc*ape.com/newsref/std/cookie_spec.html
    # # This is a generated file!  Do not edit.
 
    # .google.com   TRUE    /   FALSE   1421308121  PREF    ID=0e05040dd979207c:FF=0:NW=1:TM=1358236121:LM=1358236121:S=jcFid2XgXMIhPUPl
    # .google.com.hk    TRUE    /   FALSE   1374047321  NID 67=klMI_Z5ZPWDjUYrWSUHIE_kYI77_ziJaL0kWRoUGThagME86LKY7H-MNa2wAMI_GklIwYcD8t82qPinxzLd4GLDbmWT0OVLCXhRj0wQDC57dTNAsTs4lhVR7Yjvj2tfn
    # .google.com.hk    TRUE    /   FALSE   1421308121  PREF    ID=028f8b736db06a9a:U=6ba6d080847c8de6:FF=2:LD=zh-CN:NW=1:TM=1358236121:LM=1358236121:S=_1BcC5v3G0ZojVz8
     
    print "4. read cookies from file";
    parseAndSavedCookieFile = "parsedAndSavedCookies.txt"
    parsedCookieJarFile = cookielib.MozillaCookieJar(parseAndSavedCookieFile);
    #parsedCookieJarFile = cookielib.MozillaCookieJar(cookieFilenameMozilla);
    print parsedCookieJarFile; #    parsedCookieJarFile.load(cookieFilenameMozilla);

print parsedCookieJarFile; #<_mozillacookiejar.mozillacookiejar pref="ID=5add236cafeb990c:FF=0:NW=1:TM=1358236707:LM=1358236707:S=9lhhp0W0zTCj8FVn" for="">,,]>


if __name__=="__main__":
pythonAutoHandleCookie();



总结

Python中的cookie的自动处理，比C#好多了。

至少有已有的库可供使用。虽然用法上，需要稍微多多了解后，才知道如何使用的。