heibai2006/JupyterLab
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

加入执行结果

Browse Source
This commit is contained in:
meng 2024-01-18 21:11:41 +08:00
parent ffaa8e2145
commit 90793be5b3
8 changed files with 1603 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

167
.ipynb_checkpoints/openai-checkpoint.ipynb Normal file
View File

@ -0,0 +1,167 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"id": "21b3b31c-c951-47eb-98d3-0d41f066c449",
"metadata": {},
"outputs": [],
"source": [
"# imports\n",
"import openai # for OpenAI API calls\n",
"import time # for measuring time duration of API calls"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "63724402-590a-49e6-bf77-942835432b03",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['APIError',\n",
" 'Audio',\n",
" 'ChatCompletion',\n",
" 'Completion',\n",
" 'ContextVar',\n",
" 'Customer',\n",
" 'Deployment',\n",
" 'Edit',\n",
" 'Embedding',\n",
" 'Engine',\n",
" 'ErrorObject',\n",
" 'File',\n",
" 'FineTune',\n",
" 'Image',\n",
" 'InvalidRequestError',\n",
" 'Model',\n",
" 'Moderation',\n",
" 'OpenAIError',\n",
" 'Optional',\n",
" 'TYPE_CHECKING',\n",
" '__all__',\n",
" '__annotations__',\n",
" '__builtins__',\n",
" '__cached__',\n",
" '__doc__',\n",
" '__file__',\n",
" '__loader__',\n",
" '__name__',\n",
" '__package__',\n",
" '__path__',\n",
" '__spec__',\n",
" 'aiosession',\n",
" 'api_base',\n",
" 'api_key',\n",
" 'api_key_path',\n",
" 'api_requestor',\n",
" 'api_resources',\n",
" 'api_type',\n",
" 'api_version',\n",
" 'app_info',\n",
" 'ca_bundle_path',\n",
" 'datalib',\n",
" 'debug',\n",
" 'enable_telemetry',\n",
" 'error',\n",
" 'log',\n",
" 'openai_object',\n",
" 'openai_response',\n",
" 'organization',\n",
" 'os',\n",
" 'proxy',\n",
" 'util',\n",
" 'verify_ssl_certs',\n",
" 'version']"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dir(openai)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ad154406-c78a-4e34-9ba7-e00ded084f25",
"metadata": {},
"outputs": [],
"source": [
"# record the time before the request is sent\n",
"start_time = time.time()\n",
"\n",
"# send a ChatCompletion request to count to 100\n",
"response = openai.ChatCompletion.create(\n",
" model='gpt-3.5-turbo',\n",
" messages=[\n",
" {'role': 'user', 'content': 'Count to 100, with a comma between each number and no newlines. E.g., 1, 2, 3, ...'}\n",
" ],\n",
" temperature=0,\n",
")\n",
"\n",
"# calculate the time it took to receive the response\n",
"response_time = time.time() - start_time\n",
"\n",
"# print the time delay and text received\n",
"print(f\"Full response received {response_time:.2f} seconds after request\")\n",
"print(f\"Full response received:\\n{response}\")\n",
"reply = response['choices'][0]['message']\n",
"print(f\"Extracted reply: \\n{reply}\")\n",
"\n",
"reply_content = response['choices'][0]['message']['content']\n",
"print(f\"Extracted content: \\n{reply_content}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b42e1746-f2a7-49b1-baf9-148d72e22ce7",
"metadata": {},
"outputs": [],
"source": [
"# Example of an OpenAI ChatCompletion request with stream=True\n",
"# https://platform.openai.com/docs/guides/chat\n",
"\n",
"# a ChatCompletion request\n",
"response = openai.ChatCompletion.create(\n",
" model='gpt-3.5-turbo',\n",
" messages=[\n",
" {'role': 'user', 'content': \"What's 1+1? Answer in one word.\"}\n",
" ],\n",
" temperature=0,\n",
" stream=True # this time, we set stream=True\n",
")\n",
"\n",
"for chunk in response:\n",
" print(chunk)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "openai",
"language": "python",
"name": "openai"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.16"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

287
.ipynb_checkpoints/pymongo-pymysql测试-checkpoint.ipynb Normal file
View File

File diff suppressed because one or more lines are too long

103
.ipynb_checkpoints/test-checkpoint.ipynb Normal file
View File

@ -0,0 +1,103 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"title= 青岛：迎春花灯美轮美奂 市民观赏其乐陶陶\n",
"author= []\n",
"publish_date= None\n",
"summary= \n",
"text= 2021年2月15日晚，山东青岛西海岸新区金沙滩啤酒城花灯璀璨，赏心悦目。当日，各式花灯流光溢彩，市民和游客被美轮美奂的多种牛的造型花灯和十二生肖花灯吸引，其乐陶陶，共享新春灯会带来的欢乐和艺术盛宴。\n",
"\n",
"2021年2月15日晚，山东青岛西海岸新区金沙滩啤酒城花灯璀璨，赏心悦目。当日，各式花灯流光溢彩，市民和游客被美轮美奂的多种牛的造型花灯和十二生肖花灯吸引，其乐陶陶，共享新春灯会带来的欢乐和艺术盛宴。\n",
"\n",
"2021年2月15日晚，山东青岛西海岸新区金沙滩啤酒城花灯璀璨，赏心悦目。当日，各式花灯流光溢彩，市民和游客被美轮美奂的多种牛的造型花灯和十二生肖花灯吸引，其乐陶陶，共享新春灯会带来的欢乐和艺术盛宴。\n",
"\n",
"2021年2月15日晚，山东青岛西海岸新区金沙滩啤酒城花灯璀璨，赏心悦目。当日，各式花灯流光溢彩，市民和游客被美轮美奂的多种牛的造型花灯和十二生肖花灯吸引，其乐陶陶，共享新春灯会带来的欢乐和艺术盛宴。\n",
"\n",
"2021年2月15日晚，山东青岛西海岸新区金沙滩啤酒城花灯璀璨，赏心悦目。当日，各式花灯流光溢彩，市民和游客被美轮美奂的多种牛的造型花灯和十二生肖花灯吸引，其乐陶陶，共享新春灯会带来的欢乐和艺术盛宴。\n",
"\n",
"2021年2月15日晚，山东青岛西海岸新区金沙滩啤酒城花灯璀璨，赏心悦目。当日，各式花灯流光溢彩，市民和游客被美轮美奂的多种牛的造型花灯和十二生肖花灯吸引，其乐陶陶，共享新春灯会带来的欢乐和艺术盛宴。\n",
"\n",
"2021年2月15日晚，山东青岛西海岸新区金沙滩啤酒城花灯璀璨，赏心悦目。当日，各式花灯流光溢彩，市民和游客被美轮美奂的多种牛的造型花灯和十二生肖花灯吸引，其乐陶陶，共享新春灯会带来的欢乐和艺术盛宴。\n",
"\n",
"2021年2月15日晚，山东青岛西海岸新区金沙滩啤酒城花灯璀璨，赏心悦目。当日，各式花灯流光溢彩，市民和游客被美轮美奂的多种牛的造型花灯和十二生肖花灯吸引，其乐陶陶，共享新春灯会带来的欢乐和艺术盛宴。\n",
"\n",
"2021年2月15日晚，山东青岛西海岸新区金沙滩啤酒城花灯璀璨，赏心悦目。当日，各式花灯流光溢彩，市民和游客被美轮美奂的多种牛的造型花灯和十二生肖花灯吸引，其乐陶陶，共享新春灯会带来的欢乐和艺术盛宴。\n",
"\n",
"2021年2月15日晚，山东青岛西海岸新区金沙滩啤酒城花灯璀璨，赏心悦目。当日，各式花灯流光溢彩，市民和游客被美轮美奂的多种牛的造型花灯和十二生肖花灯吸引，其乐陶陶，共享新春灯会带来的欢乐和艺术盛宴。\n",
"\n",
"2021年2月15日晚，山东青岛西海岸新区金沙滩啤酒城花灯璀璨，赏心悦目。当日，各式花灯流光溢彩，市民和游客被美轮美奂的多种牛的造型花灯和十二生肖花灯吸引，其乐陶陶，共享新春灯会带来的欢乐和艺术盛宴。\n",
"\n",
"2021年2月15日晚，山东青岛西海岸新区金沙滩啤酒城花灯璀璨，赏心悦目。当日，各式花灯流光溢彩，市民和游客被美轮美奂的多种牛的造型花灯和十二生肖花灯吸引，其乐陶陶，共享新春灯会带来的欢乐和艺术盛宴。\n",
"\n",
"责编：吕原 \n",
"\n",
"images= {'https://appimg.dzwww.com/2021/0216/602af8081c74c.jpg', 'https://appimg.dzwww.com/2021/0216/602af808209c1.jpg', 'http://hb.dzwww.com/images/hb.ico', 'http://hb.dzwww.com/images/pc/top-logo-hb.png?v=1', 'https://appapi.dzwww.com/images/icon/cities/.png', 'https://appimg.dzwww.com/2021/0216/602af80861c94.jpg', 'https://appimg.dzwww.com/2021/0216/602af8086df8a.jpg', 'https://appimg.dzwww.com/2021/0216/602af8082499d.jpg', 'http://hb.dzwww.com/images/pc/dyh.jpg?v=1', 'https://appimg.dzwww.com/share/2021/02/16/a7c665f6c127cdb.jpg', 'https://appimg.dzwww.com/2021/0216/602af80865b93.jpg', 'http://hb.dzwww.com/images/pc/code-btn2.gif', 'https://appimg.dzwww.com/2021/0216/602af8085dbd4.jpg', 'http://hb.dzwww.com/images/pc/code-btn1.gif', 'https://appimg.dzwww.com/2021/0216/602af8085ac7d.jpg', 'https://appimg.dzwww.com/2021/0216/602af8081e9a8.jpg', 'http://appimg.dzwww.com/dzcloud/20200519/6d4cb0f483c28be39a7acc4db0e5c209.jpg', 'https://appimg.dzwww.com/2021/0216/602af80828c60.jpg', 'https://uct01.dzwww.com/images/noavatar_middle.gif', 'https://appimg.dzwww.com/2021/0216/602af80822e80.jpg', 'http://hb.dzwww.com/images/pc/code.gif?v=1', 'https://appimg.dzwww.com/2021/0216/602af80872e82.jpg', 'http://hb.dzwww.com/images/pc/kfzsq.gif?v=1', 'https://appimg.dzwww.com/share/2021/02/17/cdfe65611cdecfc.jpg', 'http://hb.dzwww.com/images/pc/noavatar_middle.jpg'}\n"
]
}
],
"source": [
"import time\n",
"from newspaper import Article\n",
"\n",
"url = 'http://hb.dzwww.com/p/7882663.html'\n",
"# 创建文章对象\n",
"news = Article(url, language='zh')\n",
"# 下载网页\n",
"news.download()\n",
"## 网页解析\n",
"news.parse()\n",
"print(\"title=\",news.title)# 获取文章标题\n",
"print(\"author=\", news.authors) # 获取文章作者\n",
"print(\"publish_date=\", news.publish_date) # 获取文章日期\n",
"# 自然语言处理\n",
"#news.nlp()\n",
"#print('keywords=',news.keywords)#从文本中提取关键字\n",
"print(\"summary=\",news.summary)# 获取文章摘要\n",
"# time.sleep(30)\n",
"print(\"text=\",news.text,\"\\n\")# 获取文章正文\n",
"\n",
"#print(\"movies=\",news.movies) # 获取文章视频链接\n",
"#print(\"top_iamge=\",news.top_image) # 获取文章顶部图片地址\n",
"print(\"images=\",news.images)#从html中提取所有图像\n",
"#print(\"imgs=\",news.imgs)\n",
"#print(\"html=\",news.html)#获取html"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python [conda env:root] *",
"language": "python",
"name": "conda-root-py"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.4"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

188
.ipynb_checkpoints/同桌100数据格式化-checkpoint.ipynb Normal file
View File

@ -0,0 +1,188 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"id\n",
"title\n",
"sphoto\n",
"teacher\n",
"Photonames\n",
"url\n",
"checkfree\n",
"playtime\n",
"svisited\n",
"sequens\n",
"sdate\n",
"id\n",
"title\n",
"sphoto\n",
"teacher\n",
"Photonames\n",
"url\n",
"checkfree\n",
"playtime\n",
"svisited\n",
"sequens\n",
"sdate\n",
"id\n",
"title\n",
"sphoto\n",
"teacher\n",
"Photonames\n",
"url\n",
"checkfree\n",
"playtime\n",
"svisited\n",
"sequens\n",
"sdate\n",
"id\n",
"title\n",
"sphoto\n",
"teacher\n",
"Photonames\n",
"url\n",
"checkfree\n",
"playtime\n",
"svisited\n",
"sequens\n",
"sdate\n",
"id\n",
"title\n",
"sphoto\n",
"teacher\n",
"Photonames\n",
"url\n",
"checkfree\n",
"playtime\n",
"svisited\n",
"sequens\n",
"sdate\n"
]
}
],
"source": [
"import json\n",
"json_str='''\n",
"{\n",
" \"status\": 1,\n",
" \"responsedata\": {\n",
" \"videoclass\": \"\",\n",
" \"page\": \"1\",\n",
" \"num\": 5,\n",
" \"records\": \"14382\",\n",
" \"totalpage\": 2877,\n",
" \"rows\": [\n",
" {\n",
" \"id\": \"158075\",\n",
" \"title\": \"三年级数学下册第六章《保护大天鹅——三位数乘两位数》乘法估算\",\n",
" \"sphoto\": \"https://image.tongzhuo100.com/video/15a7d1fe2171f438a68d95d875c53eba.jpg\",\n",
" \"teacher\": \"王老师\",\n",
" \"Photonames\": \"1612859389.mp4\",\n",
" \"url\": \"http://www.tongzhuo100.com/v/2021-02/158075.html\",\n",
" \"checkfree\": \"0\",\n",
" \"playtime\": \"00:27:45\",\n",
" \"svisited\": \"1000000\",\n",
" \"sequens\": \"24\",\n",
" \"sdate\": \"1612859417\"\n",
" },\n",
" {\n",
" \"id\": \"158073\",\n",
" \"title\": \"一年级数学下册 期末复习 四\",\n",
" \"sphoto\": \"https://image.tongzhuo100.com/video/26a7d4a193afaaa874366d4263b251a7.jpg\",\n",
" \"teacher\": \"冯老师\",\n",
" \"Photonames\": \"1612857570.mp4\",\n",
" \"url\": \"http://www.tongzhuo100.com/v/2021-02/158073.html\",\n",
" \"checkfree\": \"0\",\n",
" \"playtime\": \"00:22:19\",\n",
" \"svisited\": \"1000000\",\n",
" \"sequens\": \"57\",\n",
" \"sdate\": \"1612857611\"\n",
" },\n",
" {\n",
" \"id\": \"158072\",\n",
" \"title\": \"一年级数学下册 期末复习 三\",\n",
" \"sphoto\": \"https://image.tongzhuo100.com/video/3e30ca43993fcd8ed8d014186f7d6744.jpg\",\n",
" \"teacher\": \"冯老师\",\n",
" \"Photonames\": \"1612857453.mp4\",\n",
" \"url\": \"http://www.tongzhuo100.com/v/2021-02/158072.html\",\n",
" \"checkfree\": \"0\",\n",
" \"playtime\": \"00:24:21\",\n",
" \"svisited\": \"1000000\",\n",
" \"sequens\": \"56\",\n",
" \"sdate\": \"1612857505\"\n",
" },\n",
" {\n",
" \"id\": \"158071\",\n",
" \"title\": \"一年级数学下册 期末复习 二\",\n",
" \"sphoto\": \"https://image.tongzhuo100.com/video/3bd6e906c775cbc77c8b9f613f3f155e.jpg\",\n",
" \"teacher\": \"冯老师\",\n",
" \"Photonames\": \"1612857153.mp4\",\n",
" \"url\": \"http://www.tongzhuo100.com/v/2021-02/158071.html\",\n",
" \"checkfree\": \"0\",\n",
" \"playtime\": \"00:15:32\",\n",
" \"svisited\": \"1000000\",\n",
" \"sequens\": \"55\",\n",
" \"sdate\": \"1612857384\"\n",
" },\n",
" {\n",
" \"id\": \"158070\",\n",
" \"title\": \"一年级数学下册 第六单元《100以内的加法和减法（二）》复习（2）\",\n",
" \"sphoto\": \"https://image.tongzhuo100.com/video/d87bdd7c2f1bffb2552ed6443d6409bf.jpg\",\n",
" \"teacher\": \"冯老师\",\n",
" \"Photonames\": \"1612857034.mp4\",\n",
" \"url\": \"http://www.tongzhuo100.com/v/2021-02/158070.html\",\n",
" \"checkfree\": \"0\",\n",
" \"playtime\": \"00:22:35\",\n",
" \"svisited\": \"1000000\",\n",
" \"sequens\": \"53\",\n",
" \"sdate\": \"1612857090\"\n",
" }\n",
" ]\n",
" },\n",
" \"error\": \"\"\n",
"}\n",
"'''\n",
"rows = json.loads(json_str)['responsedata']['rows']\n",
"#print (data2['data']['playarray'])\n",
"for row in rows:\n",
" for value in row:\n",
" print(row[value])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.4"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

438
.ipynb_checkpoints/微信公众号文章爬取-checkpoint.ipynb Normal file
View File

@ -0,0 +1,438 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 70,
"metadata": {},
"outputs": [],
"source": [
"# -*- coding:utf-8 -*-\n",
"import requests\n",
"import time\n",
"import csv\n",
"import json\n",
"from lxml import etree "
]
},
{
"cell_type": "code",
"execution_count": 71,
"metadata": {},
"outputs": [],
"source": [
"url=\"https://sd.dzwww.com/sdnews/default.htm\"\n",
"header={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'}"
]
},
{
"cell_type": "code",
"execution_count": 72,
"metadata": {},
"outputs": [],
"source": [
"res=requests.get(url,headers=header)\n",
"html=etree.HTML(res.text.encode(\"latin1\").decode(\"gbk\"))"
]
},
{
"cell_type": "code",
"execution_count": 73,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"50\n",
"['盘点2019展望2020|打造样板，乡村振兴全面起势', '攻坚2020|攻坚关键在项目，山东各地抓项目如何“心中有数”“手中有招”？', '将原油码头“搬”到炼化企业门口 连接黄渤海“大油龙”全线贯通', '57个果品品牌获评齐鲁放心果品 葡萄拔得头筹苹果紧随其后', '“45度让路法”风靡网络 路遇特种车辆，如何礼让', '济南网约车细则再征民意：2月1日起施行，车辆需安装定位', '对有偿补课零容忍！济南通报12起典型问题', '双招双引，看如何招得好留得住？大众日报整版经验来了', '国务院批复 青岛再添国家级功能区 直接选中这里', '济南:将项目资金分配权交给部门 把项目确定权交给基层', '不到一月，三次半路停摆！ 唐骏箱货新车“带病”出厂就“添堵”', '问政山东丨携号转网靓号不给转还乱加业务包？省通信局长：有人为设置障碍', '问政山东丨“非实名”电话卡仍在售 店主打包票“只要不违法就不会被封”', '春运期间，加开7列途经潍坊站的列车', '济南网约车门槛“有降有升”，高德等聚合平台也要承担审核责任', '青岛纺织服装行业如何互联网升级 专业人士给方案', '2020春运大幕将拉开 青岛铁路预计发送240万人次', '“天价高速费”最新进展：山西方面退还1100元，并道歉！', '2020年将有这些新大学在山东落地 青岛成为主要建校地？', '自贸区青岛片区首个平台型经济落户西海岸新区', '世界级创新中心落户高新区 五年将聚集300家企业!', '定了！济南泉城路真正变身步行街，非驻区车辆将绕行外围', '自来水管道破损致济南经十路“成河”，他们泡水里抢修一夜没合眼', '“开门红”!2020年青岛首批专项债券80亿元顺利发行', '烟台芝罘大海阳过街天桥主体将于春节前完工，预计3月底前投用', '推动市区一体融合发展，烟台全长24公里的跨区输水管网正在启动', '济南一特色小镇项目规划有进展，将形成大规模办公区', '注意啦！济南二环北路药山西路交叉口施工，K127路临时调线', '14岁女孩长期排便困难就要带着“粪兜”过活？八旬老教授say no！', '山东首个！济南西站开通滴滴网约车专属通道', '有一种冷叫妈妈觉得你冷！孩子冬天穿太多反而更易感冒', '2020年山东全面放宽落户条件 这3个市要建成垃圾分类处理系统', '女子投诉快递员不送件竟遭登门暴打！韵达快递：已开除涉事员工', '问政山东丨两年拉黑了1000多个号码 骚扰电话“阴魂不散”该咋办？', '“雪窝”烟台清雪防滑有高招！巧用环保型融雪剂保通畅', '预防流感守护健康 医生提醒：幼儿、老人、孕妇需格外注意', '东营：雪夜一男子落水 4名民警“搭人梯”救人', '退役军人事务工作表彰奖励办法出炉 有这3个常设项目', '寻找青年“政治佳”第四季获奖作品抢先看！', '一年出警586万人次！2019年济南实现街面“两抢”315天零发案', '山财原党委副书记车滨已出任德州学院党委书记', '济莱高铁全线完成招标 钢城区段已具备施工条件', '村民称“俺村进狼了偷鸡吃” 民警全副武装赶到结果……', '手机用户呈低龄化，低劣内容借移动社交隐蔽传播', '@2020年高考生！山东新高考及模拟工作权威解析来了', '淄博：隆冬季节 消防员趴在地上救助被压车底老人', '早上入院治疗晚上就能出院 山东首家省级日间病房开始运行', '山东：推动开发区体制机制改革 打造高质量发展示范区', '村民称“俺村进狼了偷鸡吃” 民警全副武装赶到结果……', '车祸受害者治病花了30多万陷困境，加害人又无力赔偿怎么办？']\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4641108.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4641186.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4641111.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4641113.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4641112.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4641320.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4641277.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4641268.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4641259.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4641330.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4688310.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4688303.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4688271.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4688253.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4688251.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4688236.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4688209.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4688208.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4688207.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4688204.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4688194.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4688183.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4688180.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4688179.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4688175.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4688166.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4688161.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4688158.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4688057.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4688042.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4688014.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4688003.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4687984.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4687966.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4687911.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4687892.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4687870.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4687834.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4687823.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4687816.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4687803.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4687802.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4687773.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4687736.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4687735.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4687732.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4687654.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4687605.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4687592.htm\n",
"https://sd.dzwww.com/sdnews/202001/d20200109_4687587.htm\n"
]
}
],
"source": [
"#获取新闻标题\n",
"news_titles=[]\n",
"news_a=html.xpath('//div[3]/ul/li/h3/a')\n",
"print(len(news_a))\n",
"for a in news_a:\n",
" news_titles.append(a.xpath('string(.)').strip())\n",
"print(news_titles)\n",
"#获取新闻链接\n",
"news_links=html.xpath('//div[3]/ul/li/h3/a/@href')\n",
"for link in news_links:\n",
" print(link)"
]
},
{
"cell_type": "code",
"execution_count": 95,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"盘点2019展望2020|打造样板，乡村振兴全面起势\n",
"2020 01/09 05:45\n",
"来源：大众日报\n",
"作者：毛鑫鑫 张 鹏 于新悦\n",
"攻坚2020|攻坚关键在项目，山东各地抓项目如何“心中有数”“手中有招”？\n",
"2020 01/09 06:17\n",
"来源：大众日报客户端\n",
"作者：禹亚宁 孙秀岭\n",
"将原油码头“搬”到炼化企业门口 连接黄渤海“大油龙”全线贯通\n",
"2020 01/09 05:47\n",
"来源：大众日报\n",
"作者：李媛　王晶\n",
"57个果品品牌获评齐鲁放心果品 葡萄拔得头筹苹果紧随其后\n",
"2020 01/09 05:50\n",
"来源：大众日报\n",
"作者：\n",
"“45度让路法”风靡网络 路遇特种车辆，如何礼让\n",
"2020 01/09 05:50\n",
"来源：大众日报\n",
"作者：马海燕　张依盟\n",
"济南网约车细则再征民意：2月1日起施行，车辆需安装定位\n",
"2020 01/09 06:43\n",
"来源：济南时报\n",
"作者：\n",
"对有偿补课零容忍！济南通报12起典型问题\n",
"2020 01/09 06:38\n",
"来源：济南日报\n",
"作者：\n",
"双招双引，看如何招得好留得住？大众日报整版经验来了\n",
"2020 01/09 06:37\n",
"来源：大众日报\n",
"作者：张晓帆 吕光社 杨国胜 张 蓓\n",
"国务院批复 青岛再添国家级功能区 直接选中这里\n",
"2020 01/09 06:36\n",
"来源：青岛新闻网\n",
"作者：\n",
"济南:将项目资金分配权交给部门 把项目确定权交给基层\n",
"2020 01/09 06:45\n",
"来源：济南日报\n",
"作者：\n",
"不到一月，三次半路停摆！ 唐骏箱货新车“带病”出厂就“添堵”\n",
"2020 01/09 21:30\n",
"来源：大众网·海报新闻\n",
"作者：石磊\n",
"问政山东丨携号转网靓号不给转还乱加业务包？省通信局长：有人为设置障碍\n",
"2020 01/09 21:29\n",
"来源：大众网·海报新闻\n",
"作者：辛振东\n",
"问政山东丨“非实名”电话卡仍在售 店主打包票“只要不违法就不会被封”\n",
"2020 01/09 21:17\n",
"来源：大众网·海报新闻\n",
"作者：张稳\n",
"春运期间，加开7列途经潍坊站的列车\n",
"2020 01/09 21:11\n",
"来源：大众日报客户端\n",
"作者：大众日报客户端记者 张蓓\n",
"济南网约车门槛“有降有升”，高德等聚合平台也要承担审核责任\n",
"2020 01/09 21:10\n",
"来源：齐鲁晚报·齐鲁壹点\n",
"作者：刘飞跃\n",
"青岛纺织服装行业如何互联网升级 专业人士给方案\n",
"2020 01/09 21:06\n",
"来源：青岛新闻网\n",
"作者：刘倩倩\n",
"2020春运大幕将拉开 青岛铁路预计发送240万人次\n",
"2020 01/09 21:02\n",
"来源：青岛新闻网\n",
"作者：江东旭\n",
"“天价高速费”最新进展：山西方面退还1100元，并道歉！\n",
"2020 01/09 21:01\n",
"来源：大众网·海报新闻\n",
"作者：解强民\n",
"2020年将有这些新大学在山东落地 青岛成为主要建校地？\n",
"2020 01/09 21:01\n",
"来源：青岛新闻网\n",
"作者：\n",
"自贸区青岛片区首个平台型经济落户西海岸新区\n",
"2020 01/09 21:01\n",
"来源：青岛新闻网\n",
"作者：陈志伟\n",
"世界级创新中心落户高新区 五年将聚集300家企业!\n",
"2020 01/09 21:00\n",
"来源：青岛新闻网\n",
"作者：任俊峰\n",
"定了！济南泉城路真正变身步行街，非驻区车辆将绕行外围\n",
"2020 01/09 20:59\n",
"来源：齐鲁晚报·齐鲁壹点\n",
"作者：张泰来 杜亚慧\n",
"自来水管道破损致济南经十路“成河”，他们泡水里抢修一夜没合眼\n",
"2020 01/09 20:59\n",
"来源：齐鲁晚报·齐鲁壹点\n",
"作者：程凌润 孙雨杨\n",
"“开门红”!2020年青岛首批专项债券80亿元顺利发行\n",
"2020 01/09 20:58\n",
"来源：青岛新闻网\n",
"作者：刘倩倩\n",
"烟台芝罘大海阳过街天桥主体将于春节前完工，预计3月底前投用\n",
"2020 01/09 20:58\n",
"来源：齐鲁晚报·齐鲁壹点\n",
"作者：张菁\n",
"推动市区一体融合发展，烟台全长24公里的跨区输水管网正在启动\n",
"2020 01/09 20:57\n",
"来源：齐鲁晚报·齐鲁壹点\n",
"作者：曲彦霖\n",
"济南一特色小镇项目规划有进展，将形成大规模办公区\n",
"2020 01/09 20:57\n",
"来源：齐鲁晚报·齐鲁壹点\n",
"作者：吕琳\n",
"注意啦！济南二环北路药山西路交叉口施工，K127路临时调线\n",
"2020 01/09 20:57\n",
"来源：齐鲁晚报·齐鲁壹点\n",
"作者：刘飞跃\n",
"14岁女孩长期排便困难就要带着“粪兜”过活？八旬老教授say no！\n",
"2020 01/09 20:38\n",
"来源：大众网·海报新闻\n",
"作者：董昊骞\n",
"山东首个！济南西站开通滴滴网约车专属通道\n",
"2020 01/09 20:35\n",
"来源：大众网·海报新闻\n",
"作者：宋冰\n",
"有一种冷叫妈妈觉得你冷！孩子冬天穿太多反而更易感冒\n",
"2020 01/09 20:31\n",
"来源：大众网·海报新闻\n",
"作者：董昊骞\n",
"2020年山东全面放宽落户条件 这3个市要建成垃圾分类处理系统\n",
"2020 01/09 20:28\n",
"来源：大众网·海报新闻\n",
"作者：张稳 吕乐\n",
"女子投诉快递员不送件竟遭登门暴打！韵达快递：已开除涉事员工\n",
"2020 01/09 20:20\n",
"来源：大众网·海报新闻\n",
"作者：\n",
"问政山东丨两年拉黑了1000多个号码 骚扰电话“阴魂不散”该咋办？\n",
"2020 01/09 20:13\n",
"来源：大众网·海报新闻\n",
"作者：张稳\n",
"“雪窝”烟台清雪防滑有高招！巧用环保型融雪剂保通畅\n",
"2020 01/09 20:05\n",
"来源：大众网·海报新闻\n",
"作者：杨瑞远\n",
"预防流感守护健康 医生提醒：幼儿、老人、孕妇需格外注意\n",
"2020 01/09 20:04\n",
"来源：大众网·海报新闻\n",
"作者：吴宝杰\n",
"东营：雪夜一男子落水 4名民警“搭人梯”救人\n",
"2020 01/09 20:02\n",
"来源：大众网·海报新闻\n",
"作者：陈丽伟\n",
"退役军人事务工作表彰奖励办法出炉 有这3个常设项目\n",
"2020 01/09 20:00\n",
"来源：大众网·海报新闻\n",
"作者：秦文\n",
"寻找青年“政治佳”第四季获奖作品抢先看！\n",
"2020 01/09 19:57\n",
"来源：大众网·海报新闻\n",
"作者：\n",
"一年出警586万人次！2019年济南实现街面“两抢”315天零发案\n",
"2020 01/09 19:55\n",
"来源：大众网·海报新闻\n",
"作者：张珈玮\n",
"山财原党委副书记车滨已出任德州学院党委书记\n",
"2020 01/09 19:51\n",
"来源：大众网·海报新闻\n",
"作者：孙杰\n",
"济莱高铁全线完成招标 钢城区段已具备施工条件\n",
"2020 01/09 19:50\n",
"来源：大众网·海报新闻\n",
"作者：解强民\n",
"村民称“俺村进狼了偷鸡吃” 民警全副武装赶到结果……\n",
"2020 01/09 19:48\n",
"来源：大众网·海报新闻\n",
"作者：解强民\n",
"手机用户呈低龄化，低劣内容借移动社交隐蔽传播\n",
"2020 01/09 19:46\n",
"来源：新时报\n",
"作者：\n",
"@2020年高考生！山东新高考及模拟工作权威解析来了\n",
"2020 01/09 19:46\n",
"来源：大众日报客户端\n",
"作者：大众日报客户端记者 王桂利\n",
"淄博：隆冬季节 消防员趴在地上救助被压车底老人\n",
"2020 01/09 19:45\n",
"来源：大众日报客户端\n",
"作者：大众日报记者 张依盟\n",
"早上入院治疗晚上就能出院 山东首家省级日间病房开始运行\n",
"2020 01/09 19:43\n",
"来源：大众网·海报新闻\n",
"作者：董昊骞\n",
"山东：推动开发区体制机制改革 打造高质量发展示范区\n",
"2020 01/09 19:35\n",
"来源：齐鲁网\n",
"作者：\n",
"村民称“俺村进狼了偷鸡吃” 民警全副武装赶到结果……\n",
"2020 01/09 19:33\n",
"来源：大众网·海报新闻\n",
"作者：\n",
"车祸受害者治病花了30多万陷困境，加害人又无力赔偿怎么办？\n",
"2020 01/09 19:30\n",
"来源：齐鲁晚报·齐鲁壹点\n",
"作者：杨璐\n"
]
}
],
"source": [
"#爬取文章内容\n",
"import re\n",
"content=[]\n",
"for news_link in news_links:\n",
" c_res = requests.get(news_link,headers=header)\n",
" c_html = etree.HTML(c_res.text.encode(\"latin1\").decode(\"gbk\"))\n",
" s = etree.tostring(c_html).decode()\n",
" #print(s)\n",
" p = c_html.xpath('//div[@id=\"news-body\"]')\n",
" c_body = p[0].xpath('string(.)').strip()\n",
" c_body = re.sub('<[^<]+?>', '', c_body).replace('\\n', '').strip()\n",
" c_body = re.sub(r'<!--(.*?)-->', '', c_body)\n",
" #print(c_body)\n",
" p = c_html.xpath('//div[@id=\"news-head\"]/h2')\n",
" c_title = p[0].xpath('string(.)').strip()\n",
" print(c_title)\n",
" p = c_html.xpath('//div[@id=\"news-side\"]/div[1]')\n",
" c_time = p[0].xpath('string(.)').strip()\n",
" c_time = c_time.replace('\\n',' ')\n",
" c_time = c_time.replace('\\t','')\n",
" print(c_time)\n",
" p = c_html.xpath('//div[@id=\"news-side\"]/div[2]')\n",
" c_laiyuan = p[0].xpath('string(.)').strip()\n",
" c_laiyuan = c_laiyuan.replace('\\n','')\n",
" c_laiyuan = c_laiyuan.replace('\\t','')\n",
" print(c_laiyuan)\n",
" p = c_html.xpath('//div[@id=\"news-side\"]/div[3]')\n",
" c_zuozhe = p[0].xpath('string(.)').strip()\n",
" c_zuozhe = c_zuozhe.replace('\\n','')\n",
" c_zuozhe = c_zuozhe.replace('\\t','')\n",
" print(c_zuozhe)"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {},
"outputs": [
{
"ename": "FileNotFoundError",
"evalue": "[Errno 2] No such file or directory: 'data/weixin_zhonyin.csv'",
"output_type": "error",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[1;31mFileNotFoundError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m<ipython-input-34-c48b4fb69043>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[1;31m#实现多页爬取和存入csv\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[1;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"data/weixin_zhonyin.csv\"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'a+'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mnewline\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m''\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mencoding\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'utf-8'\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mfile\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 3\u001b[0m \u001b[0mcsv_writer\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcsv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwriter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 4\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 5\u001b[0m \u001b[0mtitles\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'data/weixin_zhonyin.csv'"
]
}
],
"source": [
"#实现多页爬取和存入csv\n",
"with open(\"data/weixin_zhonyin.csv\",'a+',newline='',encoding='utf-8') as file:\n",
" csv_writer=csv.writer(file)\n",
" for i in range(2):\n",
" titles=[]\n",
" contents=[]\n",
" url=\"http://weixin.sogou.com/weixin?oq=&query={}&_sug_type_=1&sut=0&lkt=0%2C0%2C0&s_from=input&ri=1&_sug_=n&type=2&sst0=1509458456759&page={}&ie=utf8&p=40040108&dp=1&w=01015002&dr=1\".format(\"中印对峙\",i+1)\n",
" res=requests.get(url,headers=header)\n",
" html=etree.HTML(res.text)\n",
" news_a=html.xpath('//div[@class=\"news-box\"]/ul[@class=\"news-list\"]/li/div[@class=\"txt-box\"]/h3/a')\n",
" for a in news_a:\n",
" titles.append(a.xpath('string(.)').strip())\n",
"# print(titles)\n",
" links=html.xpath('//div[@class=\"news-box\"]/ul[@class=\"news-list\"]/li/div[@class=\"txt-box\"]/h3/a/@href')\n",
" for link in links:\n",
" c_res=requests.get(link,headers=header)\n",
" c_html=etree.HTML(c_res.text)\n",
" p=c_html.xpath('//div[@id=\"js_content\"]/p/text()')\n",
" if p:\n",
" contents.append('。'.join(p))\n",
" else:\n",
" p=c_html.xpath('//div[@id=\"js_content\"]/p/span/text()')\n",
" contents.append('。'.join(p))\n",
" for data in zip(titles,links,contents):\n",
" print(data)\n",
" csv_writer.writerow(list(data))\n",
" time.sleep(2)\n",
" print(\"第{}页爬取完毕\".format(i+1))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.4"
}
},
"nbformat": 4,
"nbformat_minor": 4
}

205
.ipynb_checkpoints/抖音测试用-checkpoint.ipynb Normal file
View File

@ -0,0 +1,205 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"手机型号：ONEPLUS A5010\n"
]
}
],
"source": [
"import time\n",
"import uiautomator2 as u2\n",
"\n",
"\n",
"d = u2.connect_wifi('192.168.10.136')\n",
"\n",
"print('手机型号：'+d.device_info['model'])\n",
"## 我\n",
"my = d.xpath('//*[@text=\"我\"]')\n",
"my = d.xpath('//*[@resource-id=\"com.ss.android.ugc.aweme:id/root_view\"]/android.widget.FrameLayout[5]/android.widget.RelativeLayout[1]/android.widget.RelativeLayout[1]')\n",
"my.click()\n",
"## 判断是否出现 编辑资料 出现则表示在我的页面\n",
"d.xpath('//*[@resource-id=\"com.ss.android.ugc.aweme:id/ht7\"]')\n",
"\n",
"## 我的抖音号\n",
"d.xpath('//*[@resource-id=\"com.ss.android.ugc.aweme:id/l1c\"]')\n",
"## 获赞\n",
"d.xpath('//*[@resource-id=\"com.ss.android.ugc.aweme:id/b4o\"]')\n",
"## 粉丝数\n",
"d.xpath('//*[@resource-id=\"com.ss.android.ugc.aweme:id/c__\"]')\n",
"## 我的关注\n",
"d.xpath('//*[@resource-id=\"com.ss.android.ugc.aweme:id/c-d\"]')\n",
"\n",
"## 右上角打开侧边栏按钮\n",
"info = d.xpath('//*[@resource-id=\"com.ss.android.ugc.aweme:id/ghv\"]')\n",
"info.click()\n",
"## 点击侧边栏的创作中心\n",
"d.xpath('//*[@text=\"创作者服务中心\"]').click()\n",
"\n",
"d.xpath('//*[@text=\"商品橱窗\"]').click()\n",
"\n",
"## 这个是广告 后面处理\n",
"d.xpath('//*[@text=\"稍后再看\"]').click()\n",
"\n",
"d.xpath('//*[@text=\"选品广场\"]').click()\n",
"\n",
"d.xpath('//*[@text=\"服饰内衣\"]')\n",
"## 通过右上角 搜索后面的链接判断是否进入页面 此处应该等于 链接\n",
"d.xpath('//*[@resource-id=\"app\"]/android.view.View[1]/android.view.View[1]/android.view.View[3]')\n",
"d.xpath('//*[@text=\"链接\"]')\n",
"\n",
"## 点击子分类进入商品页\n",
"d.xpath('//*[@text=\"裤子\"]')\n",
"## 可以用于判断商品列表页面是否打开\n",
"text = d.xpath('//*[@resource-id=\"app\"]/android.view.View[1]/android.view.View[1]/android.view.View[2]')\n",
"d.xpath('//*[@text=\"推荐\"]')\n",
"## 这个地方是推荐就代表打开了商品列表页面\n",
"d.xpath('//*[@resource-id=\"app\"]/android.view.View[1]/android.view.View[2]/android.view.View[1]')\n",
"## 第一个是筛选栏 所以要-1\n",
"good_num = d.xpath('//*[@resource-id=\"app\"]/android.view.View').all()\n",
"## 判断是否打开了商品详情页 如果打开页面标题应该为 商品推广信息\n",
"d.xpath('//*[@resource-id=\"com.ss.android.ugc.aweme:id/title\"]')\n",
"## 售价\n",
"m_money = d.xpath('//*[@resource-id=\"app\"]/android.view.View[1]/android.view.View[1]/android.view.View[2]/android.view.View[3]/android.view.View[3]')\n",
"store_name = d.xpath('//*[@resource-id=\"app\"]/android.view.View[1]/android.view.View[1]/android.view.View[14]/android.view.View[1]/android.view.View[1]/android.view.View[2]/android.view.View[1]')\n",
"## 预计每当赚 后面三个分别为 百十个位 小数点 小数位\n",
"lirun = d.xpath('//*[@resource-id=\"app\"]/android.view.View[1]/android.view.View[1]/android.view.View[2]/android.view.View[1]/android.view.View[3]')\n",
"## 点击联系客服\n",
"d.xpath('//*[@text=\"联系商家\"]')\n",
"## 判断是否进入客服页面\n",
"d.xpath('//*[@resource-id=\"web-im-container\"]')\n",
"## 电话按钮\n",
"d.xpath('//*[@resource-id=\"web-im-container\"]/android.view.View[1]/android.view.View[1]/android.view.View[1]/android.view.View[1]/android.view.View[2]/android.view.View[4]/android.view.View[1]')\n",
"## 出现联系商家\n",
"d.xpath('//*[@text=\"联系商家\"]')\n",
"## 这里是电话沟通 还是 查看商家联系方式\n",
"d.xpath('//android.widget.ListView/android.view.View[1]/android.view.View[2]')\n",
"## 如果是联系方式 则点击查看\n",
"d.xpath('//*[@text=\"查看商家联系方式\"]')\n",
"## 点击 复制号码\n",
"d.xpath('//*[@text=\"复制号码\"]')\n",
"## 通过剪贴板获取\n",
"print(d.clipboard)\n",
"## 可以清空剪贴板\n",
"d.set_clipboard('text', 'label')\n",
"\n",
"## 点击弹出联系方式的取消\n",
"d.xpath('//*[@text=\"取消\"]')\n",
"## 返回两次到达商品列表页\n",
"d.press(\"back\")\n",
"d.press(\"back\")\n",
"\n",
"all_type = d.xpath('//*[@resource-id=\"app\"]/android.view.View[1]/android.view.View[2]/android.view.View[1]/android.view.View[1]/android.view.View[2]')\n",
"all_type.click()\n",
"\n",
"one_type = d.xpath('//*[@text=\"食品饮料\"]')\n",
"## 滑动侧栏\n",
"\n",
"\n",
"d.HTTP_TIMEOUT = 60 # 默认值60s, http默认请求超时时间\n",
"\n",
"# 当设备掉线时，等待设备在线时长，仅当TMQ=true时有效，支持通过环境变量 WAIT_FOR_DEVICE_TIMEOUT 设置\n",
"d.WAIT_FOR_DEVICE_TIMEOUT = 70 \n",
"\n",
"d(text=\"Settings\").exists # True if exists, else False\n",
"d.exists(text=\"Settings\") # alias of above property.\n",
"\n",
"# advanced usage\n",
"d(text=\"Settings\").exists(timeout=3) # wait Settings appear in 3s, same as .wait(3)\n",
"\n",
"d(text=\"Settings\").get_text() # get widget text\n",
"d(text=\"Settings\").set_text(\"My text...\") # set the text\n",
"d(text=\"Settings\").clear_text() # clear the text\n",
"\n",
"## Swipe from the center of the UI object to its edge\n",
"\n",
"## Swipe supports 4 directions:\n",
"## left\n",
"## right\n",
"## top\n",
"## bottom\n",
"d(text=\"Settings\").swipe(\"right\")\n",
"d(text=\"Settings\").swipe(\"left\", steps=10)\n",
"d(text=\"Settings\").swipe(\"up\", steps=20) # 1 steps is about 5ms, so 20 steps is about 0.1s\n",
"d(text=\"Settings\").swipe(\"down\", steps=20)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# coding: utf-8\n",
"#\n",
"import uiautomator2 as u2\n",
"from uiautomator2 import Direction\n",
"\n",
"d = u2.connect('192.168.10.136')\n",
"#d.debug = True\n",
"print(d.app_current())\n",
"print(d.device_info)\n",
"#d.click(0.525, 0.151)\n",
"#d.swipe_ext(\"up\", scale=0.9)\n",
"#d.app_list_running()\n",
"#d(text=\"Settings\").swipe(\"down\", steps=20)\n",
"\n",
"stores = d.xpath('//*[@resource-id=\"app\"]/android.view.View').all()\n",
"#store_count = count(stores)\n",
"#print(store_count)\n",
"#d.swipe_points([(200,1000),(250,500),(250,100)],0.2)\n",
"\n",
"\n",
"for store in stores:\n",
" good_attrib = store.elem.find('./android.view.View[2]')\n",
" good_name = good_attrib.attrib['text']\n",
" print(good_name)\n",
" if good_name != '':\n",
" store.click()\n",
" \n",
" title = d.xpath('//*[@resource-id=\"com.ss.android.ugc.aweme:id/title\"]').get_text()\n",
" z_money1 = d.xpath('//*[@text=\"预估每单赚\"]').parent().elem.find('./android.view.View[3]').attrib['text']\n",
" z_money2 = d.xpath('//*[@text=\"预估每单赚\"]').parent().elem.find('./android.view.View[5]').attrib['text']\n",
" z_money = z_money1 + '.' + z_money2\n",
" m_money = d.xpath('//*[@text=\"售价\"]').parent().elem.find('./android.view.View[3]').attrib['text']\n",
" d.xpath('//*[@resource-id=\"app\"]/android.view.View[1]').scroll_to(\"进店逛逛\")\n",
" store_name = d.xpath('//*[@text=\"进店逛逛\"]').parent().elem.find('./android.view.View[1]/android.view.View[2]').attrib['text']\n",
" print(z_money,m_money,store_name)\n",
" d(resourceId=\"com.ss.android.ugc.aweme:id/bbf\").click()\n",
" # 按下back键的常规方式\n",
" #d.swipe_ext(\"down\")\n",
"\n",
"#print(d(resourceId=\"//*[@resource-id=\"wall-warper\"]/android.view.View[2]/android.view.View[1]/android.widget.Button\").count)\n",
"#d.send_keys(\"back\", clear=True)\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python [conda env:root] *",
"language": "python",
"name": "conda-root-py"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.4"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

84
.ipynb_checkpoints/测试-checkpoint.ipynb Normal file
View File

@ -0,0 +1,84 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"https://v92-dy.ixigua.com/65e0f20a39ff8801b1a75cb13772f885/5f9b1f3e/video/tos/cn/tos-cn-ve-15/05235eb3a2bd4bf4b39c3e613312edb6/?a=1128&br=2649&bt=883&cr=0&cs=0&cv=1&dr=0&ds=6&er=&l=202010300259290101980670282B3A923F&lr=&mime_type=video_mp4&qs=0&rc=M3hkZjhmMzM7djMzN2kzM0ApaDxoZ2ZoNjs5Nzg3NzY5PGdgMzJhcGVvczFfLS0zLS9zczUyNTNgNjEuYTJhYzA2LjA6Yw%3D%3D&vl=&vr=\n"
]
}
],
"source": [
"import re\n",
"\n",
"import requests\n",
"\n",
"headers = {\n",
" 'User-Agent': 'User-Agent\", \"Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/16D57 Version/12.0 Safari/604.1',\n",
"}\n",
"\n",
"url = 'https://v.douyin.com/J8X86mA/' # 抖音分享的链接，自行更改\n",
"response = requests.get(url, headers=headers)\n",
"\n",
"now_url = response.url\n",
"pat_item_ids = '/video/(.*?)/'\n",
"item_ids = re.compile(pat_item_ids, re.S).findall(now_url)\n",
"\n",
"pat_dytk = 'dytk: \"(.*?)\"'\n",
"dytk = re.compile(pat_dytk, re.S).findall(response.text)\n",
"\n",
"url = 'https://www.iesdouyin.com/web/api/v2/aweme/iteminfo/'\n",
"params = {\n",
" 'item_ids': item_ids,\n",
" 'dytk': dytk\n",
"}\n",
"response = requests.get(url, headers=headers, params=params).json()\n",
"true_url = response['item_list'][0]['video']['play_addr']['url_list'][0]\n",
"true_url = true_url.replace('playwm','play')\n",
"response = requests.get(true_url, headers=headers)\n",
"true_url = response.url\n",
"print(true_url)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.4"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

131
.ipynb_checkpoints/阿里云DDNS-checkpoint.ipynb Normal file
View File

@ -0,0 +1,131 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "75e8667f-b8d9-4b1d-a119-9babdfc24947",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Looking in indexes: https://mirrors.aliyun.com/pypi/simple/, https://pypi.ngc.nvidia.com\n",
"Collecting aliyun-python-sdk-core\n",
" Downloading https://mirrors.aliyun.com/pypi/packages/1e/e3/8623c0305022610466ded2b0010a7221e9585046116263dd27cb2e56df36/aliyun-python-sdk-core-2.14.0.tar.gz (443 kB)\n",
" -------------------------------------- 443.0/443.0 kB 5.5 MB/s eta 0:00:00\n",
" Preparing metadata (setup.py): started\n",
" Preparing metadata (setup.py): finished with status 'done'\n",
"Collecting jmespath<1.0.0,>=0.9.3\n",
" Downloading https://mirrors.aliyun.com/pypi/packages/07/cb/5f001272b6faeb23c1c9e0acc04d48eaaf5c862c17709d20e3469c6e0139/jmespath-0.10.0-py2.py3-none-any.whl (24 kB)\n",
"Requirement already satisfied: cryptography>=2.6.0 in d:\\miniconda3\\lib\\site-packages (from aliyun-python-sdk-core) (41.0.7)\n",
"Requirement already satisfied: cffi>=1.12 in d:\\miniconda3\\lib\\site-packages (from cryptography>=2.6.0->aliyun-python-sdk-core) (1.16.0)\n",
"Requirement already satisfied: pycparser in d:\\miniconda3\\lib\\site-packages (from cffi>=1.12->cryptography>=2.6.0->aliyun-python-sdk-core) (2.21)\n",
"Building wheels for collected packages: aliyun-python-sdk-core\n",
" Building wheel for aliyun-python-sdk-core (setup.py): started\n",
" Building wheel for aliyun-python-sdk-core (setup.py): finished with status 'done'\n",
" Created wheel for aliyun-python-sdk-core: filename=aliyun_python_sdk_core-2.14.0-py3-none-any.whl size=535295 sha256=319f4d1ebfbba997c6937d76897cfabf74107f17cf39443b590782bb0767cd05\n",
" Stored in directory: C:\\Users\\meng\\AppData\\Local\\Temp\\pip-ephem-wheel-cache-1m9gv2st\\wheels\\b7\\c5\\b1\\1864d88c500f3e74759e9a607a3d2274d70b9409337da7df81\n",
"Successfully built aliyun-python-sdk-core\n",
"Installing collected packages: jmespath, aliyun-python-sdk-core\n",
"Successfully installed aliyun-python-sdk-core-2.14.0 jmespath-0.10.0\n",
"Looking in indexes: https://mirrors.aliyun.com/pypi/simple/, https://pypi.ngc.nvidia.com\n",
"Collecting aliyun-python-sdk-domain\n",
" Downloading https://mirrors.aliyun.com/pypi/packages/5b/4e/1c7026804b86a470d890f1cf1d5f0f20f6de9521cab7288d935c6d32a302/aliyun_python_sdk_domain-3.14.9-py2.py3-none-any.whl (134 kB)\n",
" -------------------------------------- 134.6/134.6 kB 4.0 MB/s eta 0:00:00\n",
"Requirement already satisfied: aliyun-python-sdk-core>=2.11.5 in d:\\miniconda3\\lib\\site-packages (from aliyun-python-sdk-domain) (2.14.0)\n",
"Requirement already satisfied: cryptography>=2.6.0 in d:\\miniconda3\\lib\\site-packages (from aliyun-python-sdk-core>=2.11.5->aliyun-python-sdk-domain) (41.0.7)\n",
"Requirement already satisfied: jmespath<1.0.0,>=0.9.3 in d:\\miniconda3\\lib\\site-packages (from aliyun-python-sdk-core>=2.11.5->aliyun-python-sdk-domain) (0.10.0)\n",
"Requirement already satisfied: cffi>=1.12 in d:\\miniconda3\\lib\\site-packages (from cryptography>=2.6.0->aliyun-python-sdk-core>=2.11.5->aliyun-python-sdk-domain) (1.16.0)\n",
"Requirement already satisfied: pycparser in d:\\miniconda3\\lib\\site-packages (from cffi>=1.12->cryptography>=2.6.0->aliyun-python-sdk-core>=2.11.5->aliyun-python-sdk-domain) (2.21)\n",
"Installing collected packages: aliyun-python-sdk-domain\n",
"Successfully installed aliyun-python-sdk-domain-3.14.9\n",
"Looking in indexes: https://mirrors.aliyun.com/pypi/simple/, https://pypi.ngc.nvidia.com\n",
"Collecting aliyun-python-sdk-alidns\n",
" Downloading https://mirrors.aliyun.com/pypi/packages/71/f4/2f70945b351cbffae37db518ba46f2936cc22d4d4d3969db7f4b5aac0d21/aliyun_python_sdk_alidns-3.0.7-py2.py3-none-any.whl (177 kB)\n",
" -------------------------------------- 177.1/177.1 kB 5.2 MB/s eta 0:00:00\n",
"Requirement already satisfied: aliyun-python-sdk-core>=2.11.5 in d:\\miniconda3\\lib\\site-packages (from aliyun-python-sdk-alidns) (2.14.0)\n",
"Requirement already satisfied: cryptography>=2.6.0 in d:\\miniconda3\\lib\\site-packages (from aliyun-python-sdk-core>=2.11.5->aliyun-python-sdk-alidns) (41.0.7)\n",
"Requirement already satisfied: jmespath<1.0.0,>=0.9.3 in d:\\miniconda3\\lib\\site-packages (from aliyun-python-sdk-core>=2.11.5->aliyun-python-sdk-alidns) (0.10.0)\n",
"Requirement already satisfied: cffi>=1.12 in d:\\miniconda3\\lib\\site-packages (from cryptography>=2.6.0->aliyun-python-sdk-core>=2.11.5->aliyun-python-sdk-alidns) (1.16.0)\n",
"Requirement already satisfied: pycparser in d:\\miniconda3\\lib\\site-packages (from cffi>=1.12->cryptography>=2.6.0->aliyun-python-sdk-core>=2.11.5->aliyun-python-sdk-alidns) (2.21)\n",
"Installing collected packages: aliyun-python-sdk-alidns\n",
"Successfully installed aliyun-python-sdk-alidns-3.0.7\n"
]
}
],
"source": [
"!pip3 install aliyun-python-sdk-core\n",
"!pip3 install aliyun-python-sdk-domain\n",
"!pip3 install aliyun-python-sdk-alidns"
]
},
{
"cell_type": "code",
"execution_count": 32,
"id": "2b5c4f80-5b91-4366-814b-26af5618c099",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"域名列表 {\"Domains\":{\"Domain\":[]},\"TotalCount\":0,\"PageSize\":20,\"RequestId\":\"1C6E952B-2DAB-538E-B854-85B82203759B\",\"PageNumber\":1}\n"
]
}
],
"source": [
"# 加载核心SDK\n",
"from aliyunsdkcore.client import AcsClient\n",
"from aliyunsdkcore.acs_exception.exceptions import ClientException\n",
"from aliyunsdkcore.acs_exception.exceptions import ServerException\n",
"# 加载获取 、 新增、 更新、 删除接口\n",
"from aliyunsdkalidns.request.v20150109 import DescribeDomainsRequest,DescribeSubDomainRecordsRequest, AddDomainRecordRequest, UpdateDomainRecordRequest, DeleteDomainRecordRequest\n",
"# 加载内置模块\n",
"import json,urllib\n",
"\n",
"# AccessKey 和 Secret 建议使用 RAM 子账户的 KEY 和 SECRET 增加安全性\n",
"ID = 'LTAI5tJwE7yDqpAgsDSLN5AB'\n",
"SECRET = 'we5xvM3e27dOKNiGlyhYMFPCClFlfh'\n",
"# 地区节点 可选地区取决于你的阿里云帐号等级，普通用户只有四个，分别是杭州、上海、深圳、河北，具体参考官网API\n",
"regionId = 'cn-hangzhou'\n",
"# 配置认证信息\n",
"client = AcsClient(ID, SECRET, regionId)\n",
"request = DescribeDomainsRequest.DescribeDomainsRequest()\n",
"request.set_accept_format('json')\n",
"response = client.do_action_with_exception(request)\n",
"response = str(response, encoding='utf-8')\n",
"print('域名列表',response)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b5d8bb40-db6f-4691-ba43-77128f853d36",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.18"
}
},
"nbformat": 4,
"nbformat_minor": 5
}