In [1]:
import json
import re
import chardet
import requests
In [2]:
def get_text_from_html(url):
"""
获取网页内容,匹配有地址的文本
:param url: 获取地址的url
:return: 返回包含地址的网页内容
"""
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36'
}
resp = requests.get(url, headers=headers)
content = resp.text
prefix_ip = 'var ip_result = '
for row in re.split('\\r\\n\\t', content):
if prefix_ip in row and row:
return row
In [3]:
ip = '180.97.125.228'
url = 'https://www.ip138.com/iplookup.asp?ip=%s&action=1' % ip
row = get_text_from_html(url)
row
Out [3]:
'\\t\\tvar ip_result = {"ASN¹éÊôµØ":"½\\xadËÕÊ¡ÄϾ©ÊÐ µçÐÅ ", "iP¶Î":"180.97.107.0 - 180.97.135.255", "¼æÈÝIPv6µØÖ·":"::B461:7DE4", "Ó³ÉäIPv6µØÖ·":"::FFFF:B461:7DE4", "ip_c_list":[{"begin":3026283264, "end":3026290687, "ct":"Öйú", "prov":"½\\xadËÕÊ¡", "city":"ÄϾ©ÊÐ", "area":"", "idc":"", "yunyin":"µçÐÅ", "net":""}], "zg":1};'
In [4]:
# 这个网页文字的编码是unicode形式的gbk编码内容
# Unicode codepoints U+0000 to U+00FF和latin-1一一对应,编码后保留了等价的字节流数据。
row_byte = row.encode("latin-1")
row_byte
Out [4]:
b'\\t\\tvar ip_result = {"ASN\\xb9\\xe9\\xca\\xf4\\xb5\\xd8":"\\xbd\\xad\\xcb\\xd5\\xca\\xa1\\xc4\\xcf\\xbe\\xa9\\xca\\xd0 \\xb5\\xe7\\xd0\\xc5 ", "iP\\xb6\\xce":"180.97.107.0 - 180.97.135.255", "\\xbc\\xe6\\xc8\\xddIPv6\\xb5\\xd8\\xd6\\xb7":"::B461:7DE4", "\\xd3\\xb3\\xc9\\xe4IPv6\\xb5\\xd8\\xd6\\xb7":"::FFFF:B461:7DE4", "ip_c_list":[{"begin":3026283264, "end":3026290687, "ct":"\\xd6\\xd0\\xb9\\xfa", "prov":"\\xbd\\xad\\xcb\\xd5\\xca\\xa1", "city":"\\xc4\\xcf\\xbe\\xa9\\xca\\xd0", "area":"", "idc":"", "yunyin":"\\xb5\\xe7\\xd0\\xc5", "net":""}], "zg":1};'
In [5]:
# 通过字节码获取当前的编码为gb2312
encoding_detect = chardet.detect(row_byte)
encoding_detect
Out [5]:
{'encoding': 'GB2312', 'confidence': 0.99, 'language': 'Chinese'}
In [6]:
# 因此对其进行解码即可还原最初的unicode字符。
row_target = row_byte.decode("gb2312")
row_target
Out [6]:
'\\t\\tvar ip_result = {"ASN归属地":"江苏省南京市 电信 ", "iP段":"180.97.107.0 - 180.97.135.255", "兼容IPv6地址":"::B461:7DE4", "映射IPv6地址":"::FFFF:B461:7DE4", "ip_c_list":[{"begin":3026283264, "end":3026290687, "ct":"中国", "prov":"江苏省", "city":"南京市", "area":"", "idc":"", "yunyin":"电信", "net":""}], "zg":1};'
In [7]:
# 截取文本,转化为规范的json字符串
json_str = row_target.strip().replace('var ip_result = ', '').replace(';', '')
json_str
Out [7]:
'{"ASN归属地":"江苏省南京市 电信 ", "iP段":"180.97.107.0 - 180.97.135.255", "兼容IPv6地址":"::B461:7DE4", "映射IPv6地址":"::FFFF:B461:7DE4", "ip_c_list":[{"begin":3026283264, "end":3026290687, "ct":"中国", "prov":"江苏省", "city":"南京市", "area":"", "idc":"", "yunyin":"电信", "net":""}], "zg":1}'
In [8]:
# 转化为python里面的数据格式:字典
location_dict = json.loads(json_str)
location_dict
Out [8]:
{'ASN归属地': '江苏省南京市 电信 ',
'iP段': '180.97.107.0 - 180.97.135.255',
'兼容IPv6地址': '::B461:7DE4',
'映射IPv6地址': '::FFFF:B461:7DE4',
'ip_c_list': [{'begin': 3026283264,
'end': 3026290687,
'ct': '中国',
'prov': '江苏省',
'city': '南京市',
'area': '',
'idc': '',
'yunyin': '电信',
'net': ''}],
'zg': 1}
In [9]:
# 获取中文地址
location = location_dict.get('ASN归属地', '').strip()
location
Out [9]:
'江苏省南京市 电信'
全部0条评论
快来发表一下你的评论吧 !