I. 信頼できないSSL証明書を持つサイトの処理

SSL証明書 電子証明書の一種 SSLサーバー証明書 SSLプロトコルに準拠 サーバー認証とデータ転送の暗号化を行う
クロールの際に、クロールしたいWebサイトにSSL証明書がないことを示すこのエラー（SSLError）に遭遇することがあります。
処理：res = requests.get(url,verify=False)を実行します。

II. クッキー

ユーザー情報の記録による識別
1 ログインのシミュレーション

#RenRen stays logged in

import requests
url = 'http://www.renren.com/976686556/profile' # Main personal interface
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.128 Safari/537.36',
    'Cookie':'anonymid=knvqe21amc6ghy; depovince=ZGQT; _r01_=1; taihe_bi\
    _sdk_uid=c2bd353cea6830a73eb74760fbc9fd5c; taihe_bi_sdk_session=9a91c\
    62f18e74ee26c3145bb49b4eb9e; ick_login=286c45d0-e571-4fb7-918a-46a9706\
    18110; first_login_flag=1; ln_uact=17315371375; ln_hurl=http://head.xiao\
    nei.com/photos/0/0/men_main.gif; wp_fold=0; jebecookies=ee811760-7bc0-43a9-\
    883c-0d041cb1baf0|||||; _de=A4C6B1A20CD5F525F9DA27654C2D2FDA; p=f5239823cd0af743a5f015652568b6036; t= 42783075a815b6cef9f651ca18ff5c166; societyguester=42783075a815b6cef9f651ca18ff5c166; id=976686556; xnsid=f72459d7; ver=7.0; loginfrom= null'
}
res = requests.get(url,headers=headers) # res response object
# html = res.text
with open('rr.html','w',encoding='utf-8') as file_obj:
    file_obj.write(res.text)

2 クロール防止機構

#12306 ticket search
import requests
# import json json.loads -- json type str -> python type dictionary
def query():
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.128 Safari/537.36',
        'Cookie':'_uab_collina=159490169403897938828076; JSESSIONID=090F384AC50BE0F1AFA3892BE3F6DBE9; _jc_save_wfdc_flag=dc; _jc_save_ fromStation=%u957F%u6C99%2CCSQ; _jc_save_toStation=%u5317%u4EAC%2CBJP; RAIL_DEVICEID=bbXqzYOPTc-SPgujxnGkCBr9t3sq0JQoMSYUdg- FxjyQ5IkfcPCNoreXmBAIh2HSrM9Z9awDR5onIQwy4EZ8pAhaGXWYBAH6etIlFc4dyxLudz525GAcRgVX5HLIxOE1orODUNSb9wvTBAJptPms1z5Pz5K6FXES; RAIL_ EXPIRATION=1619479086609; _jc_save_toDate=2021-04-23; BIGipServerpool_passport=182714890.50215.0000; route= 6f50b51faa11b987e576cdb301e545c4; _jc_save_fromDate=2021-04-26; BIGipServerportal=3067347210.16671.0000; BIGipServerotn= 1725497610.50210.0000'
    }
    response = requests.get('https://kyfw.12306.cn/otn/leftTicket/query?leftTicketDTO.train_date=2021-\
    04-26&leftTicketDTO.from_station=CSQ&leftTicketDTO.to_station=BJP&purpose_codes=ADULT',headers=headers)

    # print(response.content.decode('utf-8'))
    return response.json()['data']['result']


for i in query():
    # print(i)
    tem_list = i.split('|')
    # define a token to mark each piece of data
    # j = 0 Technical special
    # for n in tem_list:
    # print(j,n)
    # j += 1
    # We know from the above test that the list is the data with subscript index 3 and the soft sleep is the data with subscript index 23
    if tem_list[23] ! = 'none' and tem_list[23] ! = '':
        print(tem_list[3],'ticket available',tem_list[23])
    else:
        print(tem_list[3],'no ticket')

iii. セッション

セッションとクッキーの機能は同じです。セッションとクッキーの違いは、セッションはサーバー側で記録され、クッキーはクライアント側で記録されることです。
では、安全性を確保しつつ、グローバルに簡単に情報を読み取るにはどうしたらよいのでしょうか。そこで今回、新しいストレージ・セッションの仕組みとして誕生したのが、セッション

#Breaking the 12306 captcha
import requests
req = requests.session() # Maintain a session

def login():
    # laptop win7 python3.6
    # Get the captcha image
    pic_response = req.get('https://kyfw.12306.cn/passport/captcha/captcha-image?login_site=E&module=login&rand=sjrand')

    codeImage = pic_response.content
    fn = open('code2.png','wb')
    fn.write(codeImage)
    fn.close()
    # From the top left corner of the captcha image (0,0)
    codeStr = input('Please enter the CAPTCHA coordinates:')

    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.128 Safari/537.36'
    }
    data = {
        'answer': codeStr,
        'rand': 'sjrand',
        'login_site': 'E'
    }

    response = req.post('https://kyfw.12306.cn/passport/captcha/captcha-check',data=data,headers=headers)

    print(response.text)

login()

base64 擬似暗号化

暗号化アルゴリズムとは全く関係なく、データがより暗号文に近く見えるだけです
任意のバイナリデータを64文字で表現する方法
A-Z A-Z 0 - 9 + / この64文字を暗号化に使用します。

import base64
url = '9j/4AAQSkZJRgABAgAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/ 2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAC+ASUDASIAAhEBAxEB/ 8QAHwAAAQUBAQEBAQEAAAAAAAAAAAAAECAwQFBgcICQoL/ 8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4 +Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQBAQAAAAAAAAAAECAwQFBgcICQoL/ 8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3 +Pn6/9oADAMBAAIRAxEAPwD3+ivPNS1bUJdPlW2XWIJZ550EExgZ4mwMplZDkA5IIJwGA7Vd8P63d2Wi39zqC3k32C3VmR9gYkKSQPmJyeMZxQB21FcPqV14igvb/ Vfs2qWlklsh8qKS1fGzeWbDk9iOnpU+ r6tqVsohtdYij2W48w3GiT3DuxGdweJ0QcEcAcEHnsADsaK4Xwrq2p3un6fBd6zHIk1oqjydGuIpQxQYbzndkyPUrg0zXZdR0fxLpVqmq65c2k9rdTTpbpC8i+ W0IDAbMkASNkAEnjAoA72iuH1C6iNlpk1tr11d2lxcPula7WDpE+FLoF24YDIIyCMYzxXKXOoapB4f1W4k1PUY5LfT7qaOctcxqZlVygjJkZWA25ywGRt4OTgA9jorh/ Eev3507xBFb3OnWwtN0S75mWU/u1bcMdPvcfSpdS8RahBZ6lEtxYNLHps1zHNZuWKMm0DIOR/F+lKTsrl04OpNQW7djs6K8t/te+ WGCAXOvLM9zsuws0MsxHkGUeWfuKMEE+2e9Ra/4hktvDVguma1qkEt+ gWOC9MJdkZjmV5D90EHAO4AYHTBrneJik3Y9eOSVZTjBSXvPz89dL9vu7Hq9FeZaHrl5LqmnaWNcvCsjeWn76yuOFUthim5uQOp596ojxbq41DUzFqFrK90lwDAWZfsQh+ VW64GRljgZJFH1mNr2BZHWcnFSW1+vd+Wmz+63VHrMjFY2YKWIGQoxk+ 3NUrqVUjYsu7A3BfUjkVgeFb3UvPvtLvr2C9Sxt7dormNWzKHDHcxLHJwo596xfiDqSwaTArPKJXmTaYi6nggt8oIz8oPBNbwlzK55mIoOhUdNu+33NXX4Mt/ 8JpYzR7por+AKoacfZ2YRZB+Vio47Nn3HNXbXXNN1PcLK8hnZQCyo43KPcdRXjuqanNeK+ZZUF2TNIo67XbagOGBPyhVPXp0rUj1S5j0TUrqS4k+1OywJKpJJCcL7/ fZqowO91LxFYaeXSWR3lQZZIo2YqM98A449cVVk8Q2K6bHe3Mn2SNwSq3GFY/hz9a83nkEkkcCfbrm1UF2BXyQ0mRgnoT35OT0qCWaUab9ghIjiuLgmUqcg8/ d98KOfpQB3sPimwmtYZZC2+WLzMQqZBGM/wARUHHcdualh1SzvmZbWfzSv3sKR3rgI9UuRdvdvetEZAULIqlWCgY657l+nrXWaVc3ctmDdEbyckAbcjPynHrg/ rQB6boMirotvyxJD8844c/gOv4/ hVRPEVjd6zPp0LO0sEZZnH3Cd2Co9SCOfSqcInl8JxwW832eSQMDKFyVBY5I98dD2rn7qODTby2vEnS1gt42iKtwHDHPJJ65596ANiXxboonngnujbyI+ 1xco0YDYBGN3HTBGPXNRyeJdGZlRdStXdyAqLICWPbAHWvPLbVXO+ 8Muo28t07TF4gJUYMePlw2MDA6DpV3Rr4rDeXzM0zvIQrmMKxVRjGAB33du9AHS6h4n0q1n8s3HmygldsKGQ59OOh4z+FZkXjbT3jSacTW/wAwU74CVDDsTjBP/ wBevN9SvRLeAhMRISqLIVPJ5JOdwJ65OByabYXKxwlHgt5M/wALsAfqOP60AfUekyxzaNYyxOHje3jZWHRgVGDVysvw1j/ hFdHwu0fYYcKDnHyDjNalABRRRQAUUUUAFFFFABRRRQByNx4PuL3UfNu7yJrX7XLcLEIEbYGXA++ rBie5wMcY7kw6b4V1GLTtStLiLTok1CdFliXbKnkAYcYEUalmGRgrgZzk4xXXedJ/z7S/mv8A8VR50n/PtL+a/ wDxVAHGj4a6KSUfSdEMTNcKSNLgDBH5jIIT7yfdHYjrk1pnT9fjlSdDp80r2EdtOGkeNRIpYllAU8Hd09q3/Ok/59pfzX/4qjzpP+faX81/+KoA5/ SNL1q2u9JW9WyFtYWT25aCZ2Z2xGASpUD+A9+9XrvSp5/Fml6qrRiC0tLqB1JO4tI0JUgYxj922ee461pedJ/z7S/mv/xVHnSf8+0v5r/ 8VQBla3pd5dyWL6cbeJoJpHk8wsuQ0bqSCvO7LA5rmb7wZr8unaxb29/ZFtRsZrRlmUYJdSAxcJv4yepI56V3fnSf8+0v5r/8VR50n/PtL+a// FUAZWueH7XUdJ1GKCztftV1Gw8x4xkuQACTjPQDn2pus+Hob3R762sIbW1urm3aATeUBhWxkHHY4rX86T/n2l/Nf/iqPOk/59pfzX/4qk1dWZdObpzU47rU51/ CVvDqNtLYQW1ta28E2Io02l5nUIGOO23d+dV7jwlNc+GNG00tClzaNbCeVSQSkZ+YKcdeTjIrqvOk/wCfaX81/wDiqPOk/wCfaX81/wDiqj2MNTqWYYhcr5tV/wAH/NnJQ +ELyDxVZXqXIawtHZ182YvIxKFcbdoA5J5yah03wjq9nqtvcT3NhNbQm82whGyPOOQCf4h69Mds12fnSf8APtL+a/8AxVHnSf8APtL+a/8AxVT7CH9f15FvNMQ1Z22tt6/ j7zOa0TQ7rSjqN1f/AGGA3KwQpBZ58uNI8gDLAZJ3elZfiawXUrZoiSY3HVT1H1rtpnkkiZRbS5Puv+NZlxYTzD/j2J5H3mX/ABrSMVFWRyV60q83Unvp+CseTX+ gM7B44oRMpGxnj3bQOg68VB/Yl
img_data = base64.b64decode(url) # return is binary data
print(type(img_data))
fn = open('code.png','wb')
fn.write(img_data)
fn.close()

'''
We have opened an image data with base64 encryption

'''