From 90793be5b3ac7b32faf209ff8802fa5b9908642d Mon Sep 17 00:00:00 2001 From: meng Date: Thu, 18 Jan 2024 21:11:41 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E6=89=A7=E8=A1=8C=E7=BB=93?= =?UTF-8?q?=E6=9E=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .ipynb_checkpoints/openai-checkpoint.ipynb | 167 +++++++ .../pymongo-pymysql测试-checkpoint.ipynb | 287 ++++++++++++ .ipynb_checkpoints/test-checkpoint.ipynb | 103 ++++ .../同桌100数据格式化-checkpoint.ipynb | 188 ++++++++ .../微信公众号文章爬取-checkpoint.ipynb | 438 ++++++++++++++++++ .../抖音测试用-checkpoint.ipynb | 205 ++++++++ .ipynb_checkpoints/测试-checkpoint.ipynb | 84 ++++ .../阿里云DDNS-checkpoint.ipynb | 131 ++++++ 8 files changed, 1603 insertions(+) create mode 100644 .ipynb_checkpoints/openai-checkpoint.ipynb create mode 100644 .ipynb_checkpoints/pymongo-pymysql测试-checkpoint.ipynb create mode 100644 .ipynb_checkpoints/test-checkpoint.ipynb create mode 100644 .ipynb_checkpoints/同桌100数据格式化-checkpoint.ipynb create mode 100644 .ipynb_checkpoints/微信公众号文章爬取-checkpoint.ipynb create mode 100644 .ipynb_checkpoints/抖音测试用-checkpoint.ipynb create mode 100644 .ipynb_checkpoints/测试-checkpoint.ipynb create mode 100644 .ipynb_checkpoints/阿里云DDNS-checkpoint.ipynb diff --git a/.ipynb_checkpoints/openai-checkpoint.ipynb b/.ipynb_checkpoints/openai-checkpoint.ipynb new file mode 100644 index 0000000..02145e9 --- /dev/null +++ b/.ipynb_checkpoints/openai-checkpoint.ipynb @@ -0,0 +1,167 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "id": "21b3b31c-c951-47eb-98d3-0d41f066c449", + "metadata": {}, + "outputs": [], + "source": [ + "# imports\n", + "import openai # for OpenAI API calls\n", + "import time # for measuring time duration of API calls" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "63724402-590a-49e6-bf77-942835432b03", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['APIError',\n", + " 'Audio',\n", + " 'ChatCompletion',\n", + " 'Completion',\n", + " 'ContextVar',\n", + " 'Customer',\n", + " 'Deployment',\n", + " 'Edit',\n", + " 'Embedding',\n", + " 'Engine',\n", + " 'ErrorObject',\n", + " 'File',\n", + " 'FineTune',\n", + " 'Image',\n", + " 'InvalidRequestError',\n", + " 'Model',\n", + " 'Moderation',\n", + " 'OpenAIError',\n", + " 'Optional',\n", + " 'TYPE_CHECKING',\n", + " '__all__',\n", + " '__annotations__',\n", + " '__builtins__',\n", + " '__cached__',\n", + " '__doc__',\n", + " '__file__',\n", + " '__loader__',\n", + " '__name__',\n", + " '__package__',\n", + " '__path__',\n", + " '__spec__',\n", + " 'aiosession',\n", + " 'api_base',\n", + " 'api_key',\n", + " 'api_key_path',\n", + " 'api_requestor',\n", + " 'api_resources',\n", + " 'api_type',\n", + " 'api_version',\n", + " 'app_info',\n", + " 'ca_bundle_path',\n", + " 'datalib',\n", + " 'debug',\n", + " 'enable_telemetry',\n", + " 'error',\n", + " 'log',\n", + " 'openai_object',\n", + " 'openai_response',\n", + " 'organization',\n", + " 'os',\n", + " 'proxy',\n", + " 'util',\n", + " 'verify_ssl_certs',\n", + " 'version']" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dir(openai)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ad154406-c78a-4e34-9ba7-e00ded084f25", + "metadata": {}, + "outputs": [], + "source": [ + "# record the time before the request is sent\n", + "start_time = time.time()\n", + "\n", + "# send a ChatCompletion request to count to 100\n", + "response = openai.ChatCompletion.create(\n", + " model='gpt-3.5-turbo',\n", + " messages=[\n", + " {'role': 'user', 'content': 'Count to 100, with a comma between each number and no newlines. E.g., 1, 2, 3, ...'}\n", + " ],\n", + " temperature=0,\n", + ")\n", + "\n", + "# calculate the time it took to receive the response\n", + "response_time = time.time() - start_time\n", + "\n", + "# print the time delay and text received\n", + "print(f\"Full response received {response_time:.2f} seconds after request\")\n", + "print(f\"Full response received:\\n{response}\")\n", + "reply = response['choices'][0]['message']\n", + "print(f\"Extracted reply: \\n{reply}\")\n", + "\n", + "reply_content = response['choices'][0]['message']['content']\n", + "print(f\"Extracted content: \\n{reply_content}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b42e1746-f2a7-49b1-baf9-148d72e22ce7", + "metadata": {}, + "outputs": [], + "source": [ + "# Example of an OpenAI ChatCompletion request with stream=True\n", + "# https://platform.openai.com/docs/guides/chat\n", + "\n", + "# a ChatCompletion request\n", + "response = openai.ChatCompletion.create(\n", + " model='gpt-3.5-turbo',\n", + " messages=[\n", + " {'role': 'user', 'content': \"What's 1+1? Answer in one word.\"}\n", + " ],\n", + " temperature=0,\n", + " stream=True # this time, we set stream=True\n", + ")\n", + "\n", + "for chunk in response:\n", + " print(chunk)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "openai", + "language": "python", + "name": "openai" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/.ipynb_checkpoints/pymongo-pymysql测试-checkpoint.ipynb b/.ipynb_checkpoints/pymongo-pymysql测试-checkpoint.ipynb new file mode 100644 index 0000000..ea01188 --- /dev/null +++ b/.ipynb_checkpoints/pymongo-pymysql测试-checkpoint.ipynb @@ -0,0 +1,287 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## pymysql测试" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "INSERT INTO fa_tz45(\n", + "id,title,sphoto,teacher,Photonames,url,checkfree,playtime,\n", + "svisited,sequens,sdate,\n", + "type_all,play_urls,status\n", + ") VALUES (157258,\"部编版五年级数学下册 第三单元《长方体和正方体》长方体 正方体的展开图\",\"https://image.tongzhuo100.com/video/142d45ca161063a0bf57a334c630b6c9.jpg\",\"闫老师\",\"1612422434.mp4\",\"http://www.tongzhuo100.com/v/2021-02/157258.html\",\"0\",\"00:06:00\",\"1000000\",\"14\",\"1612422500\",\"人教版,五年级,数学,下学期\",\"https://hld.tongzhuo100.com/enc/small_course/1G1A4AA4D4/1/1G1A4AA4D4-enc.m3u8?MtsHlsUriToken=MGFiZmQ5YzNlOWI4NzJkNThjMDEzMDA1MTNmOTA5MWMlTDVtd1Bhb2tWTTdwYnl0QkdzcXZrZz09,https://hld.tongzhuo100.com/enc/small_course/1G1A4AA4D4/2/1G1A4AA4D4-enc.m3u8?MtsHlsUriToken=MGFiZmQ5YzNlOWI4NzJkNThjMDEzMDA1MTNmOTA5MWMlTDVtd1Bhb2tWTTdwYnl0QkdzcXZrZz09,https://hld.tongzhuo100.com/enc/small_course/1G1A4AA4D4/3/1G1A4AA4D4-enc.m3u8?MtsHlsUriToken=MGFiZmQ5YzNlOWI4NzJkNThjMDEzMDA1MTNmOTA5MWMlTDVtd1Bhb2tWTTdwYnl0QkdzcXZrZz09\",0)\n", + "\n", + "OK\n", + "\n", + "INSERT INTO fa_tz45(\n", + "id,title,sphoto,teacher,Photonames,url,checkfree,playtime,\n", + "svisited,sequens,sdate,\n", + "type_all,play_urls,status\n", + ") VALUES (156266,\"部编版五年级语文下册 第12课《清贫》\",\"https://image.tongzhuo100.com/video/51a723c96764b4268473dd9110c106d1.jpg\",\"陈老师\",\"1610959915.mp4\",\"http://www.tongzhuo100.com/v/2021-01/156266.html\",\"0\",\"00:07:03\",\"1000000\",\"21\",\"1610959976\",\"人教版,五年级,语文,下学期\",\"https://hld.tongzhuo100.com/enc/small_course/1G10NCNN1C/1/1G10NCNN1C-enc.m3u8?MtsHlsUriToken=MGFiZmQ5YzNlOWI4NzJkNThjMDEzMDA1MTNmOTA5MWMlL25uNzRtK1hHVlZQMFZ1VjZoTmg3Zz09,https://hld.tongzhuo100.com/enc/small_course/1G10NCNN1C/2/1G10NCNN1C-enc.m3u8?MtsHlsUriToken=MGFiZmQ5YzNlOWI4NzJkNThjMDEzMDA1MTNmOTA5MWMlL25uNzRtK1hHVlZQMFZ1VjZoTmg3Zz09,https://hld.tongzhuo100.com/enc/small_course/1G10NCNN1C/3/1G10NCNN1C-enc.m3u8?MtsHlsUriToken=MGFiZmQ5YzNlOWI4NzJkNThjMDEzMDA1MTNmOTA5MWMlL25uNzRtK1hHVlZQMFZ1VjZoTmg3Zz09\",1)\n", + "\n", + "NO\n" + ] + } + ], + "source": [ + "# -*- coding: utf-8 -*-\n", + "import pymongo\n", + "import pymysql.cursors\n", + "\n", + "#mymongo = pymongo.MongoClient(\"mongodb://localhost:27017/\")\n", + "mymongo = pymongo.MongoClient(\n", + " host= '192.168.10.172',\n", + " port= int(27017),\n", + " username='admin',\n", + " password='admin',\n", + " authSource='admin'\n", + ")\n", + "mydb = mymongo[\"crawlab_test\"]\n", + "mycol = mydb[\"tz45\"]\n", + "res = mycol.find({},{ \"_id\": 0 ,\"task_id\":0,\"type_all_json\":0,\"play_urls_json\":0}).skip(0).limit(2)\n", + "new_list = [] # 新建一个空列表用来存储元组数据\n", + "\n", + "#mysql初始化\n", + "db = pymysql.connect(host=\"192.168.10.172\",port=3307,db=\"qqbot\",user=\"root\",passwd=\"!Mfd4167208\",charset='utf8',use_unicode=True)\n", + "# 使用cursor()方法获取操作游标 \n", + "cursor = db.cursor()\n", + "# SQL 插入语句\n", + "sql = \"\"\"\n", + "INSERT INTO fa_tz45(\n", + "id,title,sphoto,teacher,Photonames,url,checkfree,playtime,\n", + "svisited,sequens,sdate,\n", + "type_all,play_urls,status\n", + ") VALUES (%d,\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",%d)\n", + "\"\"\" # 要插入的数据\n", + "\n", + "for x in res:\n", + " #print(type(x))\n", + " x['id'] = int(x['id'])\n", + " tup = tuple(x.values()) # 构造元组\n", + " #print(tup)\n", + " try:\n", + " print(sql%tup)\n", + " cursor.execute(sql%tup)\n", + " db.commit()\n", + " #cursor.close()\n", + " print(\"OK\")\n", + " except:\n", + " print(\"NO\")\n", + " # 如果发生错误则回滚\n", + " db.rollback()\n", + " new_list.append(tup) # [(),(),()...]\n", + "#INSERT INTO `qqbot`.`fa_tz45` (`id`, `title`, `sphoto`, `teacher`, `Photonames`, `url`, `checkfree`, `playtime`, `svisited`, `sequens`, `sdate`, `type_all`, `type_all_json`, `play_urls`, `play_urls_json`, `status`) VALUES (1, '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%d');\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 查询" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "ename": "ProgrammingError", + "evalue": "(1146, \"Table 'qqbot.rc_member' doesn't exist\")", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mProgrammingError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mcursor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"\"\"select * from rc_member where username LIKE %s\"\"\"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m\"%网络科技有限公司%\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[1;32mD:\\ProgramData\\Miniconda3\\lib\\site-packages\\pymysql\\cursors.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self, query, args)\u001b[0m\n\u001b[0;32m 168\u001b[0m \u001b[0mquery\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmogrify\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mquery\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 169\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 170\u001b[1;33m \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_query\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mquery\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 171\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_executed\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mquery\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 172\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32mD:\\ProgramData\\Miniconda3\\lib\\site-packages\\pymysql\\cursors.py\u001b[0m in \u001b[0;36m_query\u001b[1;34m(self, q)\u001b[0m\n\u001b[0;32m 326\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_last_executed\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mq\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 327\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_clear_result\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 328\u001b[1;33m \u001b[0mconn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mquery\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mq\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 329\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_get_result\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 330\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrowcount\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32mD:\\ProgramData\\Miniconda3\\lib\\site-packages\\pymysql\\connections.py\u001b[0m in \u001b[0;36mquery\u001b[1;34m(self, sql, unbuffered)\u001b[0m\n\u001b[0;32m 515\u001b[0m \u001b[0msql\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msql\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mencoding\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'surrogateescape'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 516\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_execute_command\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCOMMAND\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mCOM_QUERY\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msql\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 517\u001b[1;33m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_affected_rows\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_read_query_result\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0munbuffered\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0munbuffered\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 518\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_affected_rows\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 519\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32mD:\\ProgramData\\Miniconda3\\lib\\site-packages\\pymysql\\connections.py\u001b[0m in \u001b[0;36m_read_query_result\u001b[1;34m(self, unbuffered)\u001b[0m\n\u001b[0;32m 730\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 731\u001b[0m \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mMySQLResult\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 732\u001b[1;33m \u001b[0mresult\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 733\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 734\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mserver_status\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32mD:\\ProgramData\\Miniconda3\\lib\\site-packages\\pymysql\\connections.py\u001b[0m in \u001b[0;36mread\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 1073\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1074\u001b[0m \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1075\u001b[1;33m \u001b[0mfirst_packet\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconnection\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_read_packet\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1076\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1077\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mfirst_packet\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_ok_packet\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32mD:\\ProgramData\\Miniconda3\\lib\\site-packages\\pymysql\\connections.py\u001b[0m in \u001b[0;36m_read_packet\u001b[1;34m(self, packet_type)\u001b[0m\n\u001b[0;32m 682\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 683\u001b[0m \u001b[0mpacket\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpacket_type\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbuff\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mencoding\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 684\u001b[1;33m \u001b[0mpacket\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcheck_error\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 685\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mpacket\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 686\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32mD:\\ProgramData\\Miniconda3\\lib\\site-packages\\pymysql\\protocol.py\u001b[0m in \u001b[0;36mcheck_error\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 218\u001b[0m \u001b[0merrno\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_uint16\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 219\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mDEBUG\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"errno =\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merrno\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 220\u001b[1;33m \u001b[0merr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mraise_mysql_exception\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 221\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 222\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mdump\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32mD:\\ProgramData\\Miniconda3\\lib\\site-packages\\pymysql\\err.py\u001b[0m in \u001b[0;36mraise_mysql_exception\u001b[1;34m(data)\u001b[0m\n\u001b[0;32m 107\u001b[0m \u001b[0merrval\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'utf-8'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'replace'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 108\u001b[0m \u001b[0merrorclass\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0merror_map\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0merrno\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mInternalError\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 109\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0merrorclass\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0merrno\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merrval\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[1;31mProgrammingError\u001b[0m: (1146, \"Table 'qqbot.rc_member' doesn't exist\")" + ] + } + ], + "source": [ + "cursor.execute(\"\"\"select * from rc_member where username LIKE %s\"\"\",\"%网络科技有限公司%\")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "repetition = cursor.fetchall()" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "((25, '山东奥商网络科技有限公司临沂分公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779552@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '临沂兰山奥尔诺财富中心1801', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (68, '青岛翼付银通网络科技有限公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779509@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '临沂市兰山区临西七路与育才路交汇处西北角维洛美亚酒店二楼', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (69, '展跃网络科技有限公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779508@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '环球国际A座1607', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (82, '山东奥商网络科技有限公司临沂分公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779495@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '临沂兰山奥尔诺财富中心1801', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (125, '青岛翼付银通网络科技有限公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779452@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '临沂市兰山区临西七路与育才路交汇处西北角维洛美亚酒店二楼', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (126, '展跃网络科技有限公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779451@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '环球国际A座1607', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', None, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (139, '山东奥商网络科技有限公司临沂分公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779438@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '临沂兰山奥尔诺财富中心1801', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (182, '青岛翼付银通网络科技有限公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779395@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '临沂市兰山区临西七路与育才路交汇处西北角维洛美亚酒店二楼', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (183, '展跃网络科技有限公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779394@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '环球国际A座1607', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (196, '山东奥商网络科技有限公司临沂分公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779381@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '临沂兰山奥尔诺财富中心1801', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', None, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (238, '青岛翼付银通网络科技有限公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779339@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '临沂市兰山区临西七路与育才路交汇处西北角维洛美亚酒店二楼', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (239, '展跃网络科技有限公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779338@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '环球国际A座1607', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (252, '山东奥商网络科技有限公司临沂分公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779325@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '临沂兰山奥尔诺财富中心1801', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (295, '青岛翼付银通网络科技有限公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779282@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '临沂市兰山区临西七路与育才路交汇处西北角维洛美亚酒店二楼', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (296, '展跃网络科技有限公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779281@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '环球国际A座1607', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', None, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (309, '山东奥商网络科技有限公司临沂分公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779268@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '临沂兰山奥尔诺财富中心1801', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (352, '青岛翼付银通网络科技有限公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779225@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '临沂市兰山区临西七路与育才路交汇处西北角维洛美亚酒店二楼', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (353, '展跃网络科技有限公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779224@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '环球国际A座1607', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (366, '山东奥商网络科技有限公司临沂分公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779211@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '临沂兰山奥尔诺财富中心1801', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', None, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (408, '青岛翼付银通网络科技有限公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779169@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '临沂市兰山区临西七路与育才路交汇处西北角维洛美亚酒店二楼', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (409, '展跃网络科技有限公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779168@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '环球国际A座1607', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (422, '山东奥商网络科技有限公司临沂分公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779155@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '临沂兰山奥尔诺财富中心1801', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (465, '青岛翼付银通网络科技有限公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779112@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '临沂市兰山区临西七路与育才路交汇处西北角维洛美亚酒店二楼', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (466, '展跃网络科技有限公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779111@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '环球国际A座1607', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', None, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (479, '山东奥商网络科技有限公司临沂分公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779098@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '临沂兰山奥尔诺财富中心1801', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (522, '青岛翼付银通网络科技有限公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779055@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '临沂市兰山区临西七路与育才路交汇处西北角维洛美亚酒店二楼', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (523, '展跃网络科技有限公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779054@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '环球国际A座1607', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (536, '山东奥商网络科技有限公司临沂分公司', '2fedf0d69979822e454e6a1d5cfd943a', '378779041@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '临沂兰山奥尔诺财富中心1801', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', None, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (578, '青岛翼付银通网络科技有限公司', '2fedf0d69979822e454e6a1d5cfd943a', '378778999@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '临沂市兰山区临西七路与育才路交汇处西北角维洛美亚酒店二楼', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (579, '展跃网络科技有限公司', '2fedf0d69979822e454e6a1d5cfd943a', '378778998@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '环球国际A座1607', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (592, '山东奥商网络科技有限公司临沂分公司', '2fedf0d69979822e454e6a1d5cfd943a', '378778985@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '临沂兰山奥尔诺财富中心1801', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (635, '青岛翼付银通网络科技有限公司', '2fedf0d69979822e454e6a1d5cfd943a', '378778942@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '临沂市兰山区临西七路与育才路交汇处西北角维洛美亚酒店二楼', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (636, '展跃网络科技有限公司', '2fedf0d69979822e454e6a1d5cfd943a', '378778941@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '环球国际A座1607', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', None, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (649, '山东奥商网络科技有限公司临沂分公司', '2fedf0d69979822e454e6a1d5cfd943a', '378778928@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '临沂兰山奥尔诺财富中心1801', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (692, '青岛翼付银通网络科技有限公司', '2fedf0d69979822e454e6a1d5cfd943a', '378778885@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '临沂市兰山区临西七路与育才路交汇处西北角维洛美亚酒店二楼', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (693, '展跃网络科技有限公司', '2fedf0d69979822e454e6a1d5cfd943a', '378778884@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '环球国际A座1607', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', 0, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0), (706, '山东奥商网络科技有限公司临沂分公司', '2fedf0d69979822e454e6a1d5cfd943a', '378778871@qq.com', '15318537111', '127.0.0.1', 1557929151, '223.104.188.249', 1561643616, 3, 3, 'fb58d9', '临沂兰山奥尔诺财富中心1801', 0, '', 1, 0, 0, '', None, '', '', '', '', '', '', None, '', 1, 5, None, 0, 0, '', None, 1, 0, 0, None, '', '', 0))\n" + ] + } + ], + "source": [ + "print(repetition)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 插入数据" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "751\n", + "0\n" + ] + } + ], + "source": [ + "import os\n", + "from pymongo import MongoClient\n", + "import pymysql.cursors\n", + "import random\n", + "import string\n", + "\n", + "item = {'com_add': '临沂-兰陵 山东省临沂市苍山县向阳路',\n", + " 'com_address': '临沂-兰陵 山东省临沂市苍山县向阳路',\n", + " 'com_dec': '美团外面兰陵分公司期待你的加入\\n'\n", + " '易观智库发布中国互联网餐饮外卖市场2015上半年专题研究报告，报告显示，“美团外卖”整体市场订单以41.24%的份额，占据互联网外卖市场*名。美团外卖相关负责人称，这是他们重视用户 体验，秉承“消费者*”价值观的结果。',\n", + " 'com_fanwei': '兰陵县市场监督管理局',\n", + " 'com_fuli': '',\n", + " 'com_guimo': '企业未添加公司规模',\n", + " 'com_hangye': '互联网/电子商务',\n", + " 'com_jiesao': '美团外面兰陵分公司期待你的加入易观智库发布中国互联网餐饮外卖市场2015上半年专题研究报告，报告显示，“美团外卖”整体市场订单以41.24%的份额，占据互联网外卖市场*名。美团外卖相关负责人称，这是他们重视用户体验，秉承“消费者*”价值观的结果。',\n", + " 'com_manguimo': '50-99人',\n", + " 'com_name': '兰陵县云瑞电子商务有限公司',\n", + " 'com_name1': ' 兰陵县云瑞电子商务有限公司 ',\n", + " 'com_name2': '兰陵县云瑞电子商务有限公司',\n", + " 'com_teshe': '',\n", + " 'com_url': 'https://qy.58.com/16200539417351/?entinfo=41894018467719_0',\n", + " 'job_des': ' 建立并实施人力资源方针和行动计划，以支持公司达到预期经营目标。骑手的招聘和培训 ',\n", + " 'job_list_title': '配送站长',\n", + " 'job_num': ' 招若干人 ',\n", + " 'job_title': '配送站长 ',\n", + " 'job_url': 'https://linyi.58.com/renli/41894018467719x.shtml?psid=194173491208152217112869437&entinfo=41894018467719_p&ytdzwdetaildj=0&finalCp=finalCp=000001270000000000020000000000000000_194173491208152217112869437&tjfrom=pc_list_left_pt__194173491208152217112869437__153955228780822528__pt',\n", + " 'job_xinzi': '薪资面议',\n", + " 'job_xueli': '高中',\n", + " 'job_yingjie': ' 1-2年 ',\n", + " 'linkemail': '企业未添加招聘邮箱',\n", + " 'linkman': '赵恩会',\n", + " 'linktelpic': None,\n", + " 'site_link': '企业未添加公司官网',\n", + " 'task_id': None}\n", + "com_address = item['com_address'].replace(\"住所：\", '')\n", + "com_address = com_address.replace(\"-\", '')\n", + "com_address = com_address.replace(\" \", '')\n", + "if item['linkemail'] == \"企业未添加招聘邮箱\":\n", + " email = ''.join(random.sample(string.ascii_letters + string.digits, 9))\n", + " email = email + \"@des8.com\"\n", + "else:\n", + " email = item['linkemail']\n", + "connect = pymysql.connect(\n", + " host='192.168.10.100',\n", + " port=3306,\n", + " db='rencai',\n", + " user='root',\n", + " passwd='4167208',\n", + " charset='utf8',\n", + " use_unicode=True)\n", + " # 通过cursor执行增删查改\n", + "mycursor = connect.cursor()\n", + "\n", + "mycursor.execute(\"\"\"select * from rc_member where username = %s\"\"\", item['com_name'])\n", + "repetition = mycursor.fetchone()\n", + "uid = repetition[0]\n", + "print(int(uid))\n", + "mycursor.execute(\"\"\"INSERT INTO `rc_company`(`uid`, `name`, `shortname`, `hy`, `pr`, `provinceid`, `cityid`, `three_cityid`, `mun`, `sdate`, `money`, `moneytype`, `content`, `address`, `zip`, `linkman`, `linkjob`, `linkqq`, `linkphone`, `linktel`, `linkmail`, `website`, `x`, `y`, `logo`, `payd`, `integral`, `lastupdate`, `cloudtype`, `jobtime`, `r_status`, `firmpic`, `rec`, `hits`, `ant_num`, `pl_time`, `pl_status`, `order`, `admin_remark`, `email_dy`, `msg_dy`, `sync`, `hy_dy`, `moblie_status`, `email_status`, `yyzz_status`, `hottime`, `did`, `busstops`, `infostatus`, `conid`, `addtime`, `comqcode`, `crm_uid`, `crm_time`, `crm_status`, `welfare`) VALUES (%s, %s, '', NULL, NULL, NULL, NULL, NULL, NULL, '', NULL, 1, %s, %s, '', '', '', '', '', '15318537111', '', '', '', '', '', 0, 0, '', NULL, NULL, 0, '', 0, 0, 0, NULL, 1, 0, '', 0, 0, 0, '', 0, 0, 0, 0, NULL, '', 1, NULL, NULL, '', 0, NULL, 0, '');\"\"\",(int(uid),item['com_name'],item['com_dec'], com_address))\n", + "connect.commit()\n", + "print(mycursor.lastrowid)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "临沂-兰山-通达路 临沂兰山区临西一路与涑河南街交汇临商一号10楼\n", + "临沂兰山通达路 临沂兰山区临西一路与涑河南街交汇临商一号10楼\n", + "临沂兰山通达路临沂兰山区临西一路与涑河南街交汇临商一号10楼\n" + ] + } + ], + "source": [ + "a = \"住所：临沂-兰山-通达路 临沂兰山区临西一路与涑河南街交汇临商一号10楼\"\n", + "a = a.replace(\"住所：\",'')\n", + "a = a.replace(\"-\",'')\n", + "a = a.replace(\" \",'')\n", + "print(a)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/.ipynb_checkpoints/test-checkpoint.ipynb b/.ipynb_checkpoints/test-checkpoint.ipynb new file mode 100644 index 0000000..4b23d34 --- /dev/null +++ b/.ipynb_checkpoints/test-checkpoint.ipynb @@ -0,0 +1,103 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "title= 青岛：迎春花灯美轮美奂 市民观赏其乐陶陶\n", + "author= []\n", + "publish_date= None\n", + "summary= \n", + "text= 2021年2月15日晚，山东青岛西海岸新区金沙滩啤酒城花灯璀璨，赏心悦目。当日，各式花灯流光溢彩，市民和游客被美轮美奂的多种牛的造型花灯和十二生肖花灯吸引，其乐陶陶，共享新春灯会带来的欢乐和艺术盛宴。\n", + "\n", + "2021年2月15日晚，山东青岛西海岸新区金沙滩啤酒城花灯璀璨，赏心悦目。当日，各式花灯流光溢彩，市民和游客被美轮美奂的多种牛的造型花灯和十二生肖花灯吸引，其乐陶陶，共享新春灯会带来的欢乐和艺术盛宴。\n", + "\n", + "2021年2月15日晚，山东青岛西海岸新区金沙滩啤酒城花灯璀璨，赏心悦目。当日，各式花灯流光溢彩，市民和游客被美轮美奂的多种牛的造型花灯和十二生肖花灯吸引，其乐陶陶，共享新春灯会带来的欢乐和艺术盛宴。\n", + "\n", + "2021年2月15日晚，山东青岛西海岸新区金沙滩啤酒城花灯璀璨，赏心悦目。当日，各式花灯流光溢彩，市民和游客被美轮美奂的多种牛的造型花灯和十二生肖花灯吸引，其乐陶陶，共享新春灯会带来的欢乐和艺术盛宴。\n", + "\n", + "2021年2月15日晚，山东青岛西海岸新区金沙滩啤酒城花灯璀璨，赏心悦目。当日，各式花灯流光溢彩，市民和游客被美轮美奂的多种牛的造型花灯和十二生肖花灯吸引，其乐陶陶，共享新春灯会带来的欢乐和艺术盛宴。\n", + "\n", + "2021年2月15日晚，山东青岛西海岸新区金沙滩啤酒城花灯璀璨，赏心悦目。当日，各式花灯流光溢彩，市民和游客被美轮美奂的多种牛的造型花灯和十二生肖花灯吸引，其乐陶陶，共享新春灯会带来的欢乐和艺术盛宴。\n", + "\n", + "2021年2月15日晚，山东青岛西海岸新区金沙滩啤酒城花灯璀璨，赏心悦目。当日，各式花灯流光溢彩，市民和游客被美轮美奂的多种牛的造型花灯和十二生肖花灯吸引，其乐陶陶，共享新春灯会带来的欢乐和艺术盛宴。\n", + "\n", + "2021年2月15日晚，山东青岛西海岸新区金沙滩啤酒城花灯璀璨，赏心悦目。当日，各式花灯流光溢彩，市民和游客被美轮美奂的多种牛的造型花灯和十二生肖花灯吸引，其乐陶陶，共享新春灯会带来的欢乐和艺术盛宴。\n", + "\n", + "2021年2月15日晚，山东青岛西海岸新区金沙滩啤酒城花灯璀璨，赏心悦目。当日，各式花灯流光溢彩，市民和游客被美轮美奂的多种牛的造型花灯和十二生肖花灯吸引，其乐陶陶，共享新春灯会带来的欢乐和艺术盛宴。\n", + "\n", + "2021年2月15日晚，山东青岛西海岸新区金沙滩啤酒城花灯璀璨，赏心悦目。当日，各式花灯流光溢彩，市民和游客被美轮美奂的多种牛的造型花灯和十二生肖花灯吸引，其乐陶陶，共享新春灯会带来的欢乐和艺术盛宴。\n", + "\n", + "2021年2月15日晚，山东青岛西海岸新区金沙滩啤酒城花灯璀璨，赏心悦目。当日，各式花灯流光溢彩，市民和游客被美轮美奂的多种牛的造型花灯和十二生肖花灯吸引，其乐陶陶，共享新春灯会带来的欢乐和艺术盛宴。\n", + "\n", + "2021年2月15日晚，山东青岛西海岸新区金沙滩啤酒城花灯璀璨，赏心悦目。当日，各式花灯流光溢彩，市民和游客被美轮美奂的多种牛的造型花灯和十二生肖花灯吸引，其乐陶陶，共享新春灯会带来的欢乐和艺术盛宴。\n", + "\n", + "责编：吕原 \n", + "\n", + "images= {'https://appimg.dzwww.com/2021/0216/602af8081c74c.jpg', 'https://appimg.dzwww.com/2021/0216/602af808209c1.jpg', 'http://hb.dzwww.com/images/hb.ico', 'http://hb.dzwww.com/images/pc/top-logo-hb.png?v=1', 'https://appapi.dzwww.com/images/icon/cities/.png', 'https://appimg.dzwww.com/2021/0216/602af80861c94.jpg', 'https://appimg.dzwww.com/2021/0216/602af8086df8a.jpg', 'https://appimg.dzwww.com/2021/0216/602af8082499d.jpg', 'http://hb.dzwww.com/images/pc/dyh.jpg?v=1', 'https://appimg.dzwww.com/share/2021/02/16/a7c665f6c127cdb.jpg', 'https://appimg.dzwww.com/2021/0216/602af80865b93.jpg', 'http://hb.dzwww.com/images/pc/code-btn2.gif', 'https://appimg.dzwww.com/2021/0216/602af8085dbd4.jpg', 'http://hb.dzwww.com/images/pc/code-btn1.gif', 'https://appimg.dzwww.com/2021/0216/602af8085ac7d.jpg', 'https://appimg.dzwww.com/2021/0216/602af8081e9a8.jpg', 'http://appimg.dzwww.com/dzcloud/20200519/6d4cb0f483c28be39a7acc4db0e5c209.jpg', 'https://appimg.dzwww.com/2021/0216/602af80828c60.jpg', 'https://uct01.dzwww.com/images/noavatar_middle.gif', 'https://appimg.dzwww.com/2021/0216/602af80822e80.jpg', 'http://hb.dzwww.com/images/pc/code.gif?v=1', 'https://appimg.dzwww.com/2021/0216/602af80872e82.jpg', 'http://hb.dzwww.com/images/pc/kfzsq.gif?v=1', 'https://appimg.dzwww.com/share/2021/02/17/cdfe65611cdecfc.jpg', 'http://hb.dzwww.com/images/pc/noavatar_middle.jpg'}\n" + ] + } + ], + "source": [ + "import time\n", + "from newspaper import Article\n", + "\n", + "url = 'http://hb.dzwww.com/p/7882663.html'\n", + "# 创建文章对象\n", + "news = Article(url, language='zh')\n", + "# 下载网页\n", + "news.download()\n", + "## 网页解析\n", + "news.parse()\n", + "print(\"title=\",news.title)# 获取文章标题\n", + "print(\"author=\", news.authors) # 获取文章作者\n", + "print(\"publish_date=\", news.publish_date) # 获取文章日期\n", + "# 自然语言处理\n", + "#news.nlp()\n", + "#print('keywords=',news.keywords)#从文本中提取关键字\n", + "print(\"summary=\",news.summary)# 获取文章摘要\n", + "# time.sleep(30)\n", + "print(\"text=\",news.text,\"\\n\")# 获取文章正文\n", + "\n", + "#print(\"movies=\",news.movies) # 获取文章视频链接\n", + "#print(\"top_iamge=\",news.top_image) # 获取文章顶部图片地址\n", + "print(\"images=\",news.images)#从html中提取所有图像\n", + "#print(\"imgs=\",news.imgs)\n", + "#print(\"html=\",news.html)#获取html" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:root] *", + "language": "python", + "name": "conda-root-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/.ipynb_checkpoints/同桌100数据格式化-checkpoint.ipynb b/.ipynb_checkpoints/同桌100数据格式化-checkpoint.ipynb new file mode 100644 index 0000000..4dbbba9 --- /dev/null +++ b/.ipynb_checkpoints/同桌100数据格式化-checkpoint.ipynb @@ -0,0 +1,188 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "id\n", + "title\n", + "sphoto\n", + "teacher\n", + "Photonames\n", + "url\n", + "checkfree\n", + "playtime\n", + "svisited\n", + "sequens\n", + "sdate\n", + "id\n", + "title\n", + "sphoto\n", + "teacher\n", + "Photonames\n", + "url\n", + "checkfree\n", + "playtime\n", + "svisited\n", + "sequens\n", + "sdate\n", + "id\n", + "title\n", + "sphoto\n", + "teacher\n", + "Photonames\n", + "url\n", + "checkfree\n", + "playtime\n", + "svisited\n", + "sequens\n", + "sdate\n", + "id\n", + "title\n", + "sphoto\n", + "teacher\n", + "Photonames\n", + "url\n", + "checkfree\n", + "playtime\n", + "svisited\n", + "sequens\n", + "sdate\n", + "id\n", + "title\n", + "sphoto\n", + "teacher\n", + "Photonames\n", + "url\n", + "checkfree\n", + "playtime\n", + "svisited\n", + "sequens\n", + "sdate\n" + ] + } + ], + "source": [ + "import json\n", + "json_str='''\n", + "{\n", + " \"status\": 1,\n", + " \"responsedata\": {\n", + " \"videoclass\": \"\",\n", + " \"page\": \"1\",\n", + " \"num\": 5,\n", + " \"records\": \"14382\",\n", + " \"totalpage\": 2877,\n", + " \"rows\": [\n", + " {\n", + " \"id\": \"158075\",\n", + " \"title\": \"三年级数学下册第六章《保护大天鹅——三位数乘两位数》乘法估算\",\n", + " \"sphoto\": \"https://image.tongzhuo100.com/video/15a7d1fe2171f438a68d95d875c53eba.jpg\",\n", + " \"teacher\": \"王老师\",\n", + " \"Photonames\": \"1612859389.mp4\",\n", + " \"url\": \"http://www.tongzhuo100.com/v/2021-02/158075.html\",\n", + " \"checkfree\": \"0\",\n", + " \"playtime\": \"00:27:45\",\n", + " \"svisited\": \"1000000\",\n", + " \"sequens\": \"24\",\n", + " \"sdate\": \"1612859417\"\n", + " },\n", + " {\n", + " \"id\": \"158073\",\n", + " \"title\": \"一年级数学下册 期末复习 四\",\n", + " \"sphoto\": \"https://image.tongzhuo100.com/video/26a7d4a193afaaa874366d4263b251a7.jpg\",\n", + " \"teacher\": \"冯老师\",\n", + " \"Photonames\": \"1612857570.mp4\",\n", + " \"url\": \"http://www.tongzhuo100.com/v/2021-02/158073.html\",\n", + " \"checkfree\": \"0\",\n", + " \"playtime\": \"00:22:19\",\n", + " \"svisited\": \"1000000\",\n", + " \"sequens\": \"57\",\n", + " \"sdate\": \"1612857611\"\n", + " },\n", + " {\n", + " \"id\": \"158072\",\n", + " \"title\": \"一年级数学下册 期末复习 三\",\n", + " \"sphoto\": \"https://image.tongzhuo100.com/video/3e30ca43993fcd8ed8d014186f7d6744.jpg\",\n", + " \"teacher\": \"冯老师\",\n", + " \"Photonames\": \"1612857453.mp4\",\n", + " \"url\": \"http://www.tongzhuo100.com/v/2021-02/158072.html\",\n", + " \"checkfree\": \"0\",\n", + " \"playtime\": \"00:24:21\",\n", + " \"svisited\": \"1000000\",\n", + " \"sequens\": \"56\",\n", + " \"sdate\": \"1612857505\"\n", + " },\n", + " {\n", + " \"id\": \"158071\",\n", + " \"title\": \"一年级数学下册 期末复习 二\",\n", + " \"sphoto\": \"https://image.tongzhuo100.com/video/3bd6e906c775cbc77c8b9f613f3f155e.jpg\",\n", + " \"teacher\": \"冯老师\",\n", + " \"Photonames\": \"1612857153.mp4\",\n", + " \"url\": \"http://www.tongzhuo100.com/v/2021-02/158071.html\",\n", + " \"checkfree\": \"0\",\n", + " \"playtime\": \"00:15:32\",\n", + " \"svisited\": \"1000000\",\n", + " \"sequens\": \"55\",\n", + " \"sdate\": \"1612857384\"\n", + " },\n", + " {\n", + " \"id\": \"158070\",\n", + " \"title\": \"一年级数学下册 第六单元《100以内的加法和减法（二）》复习（2）\",\n", + " \"sphoto\": \"https://image.tongzhuo100.com/video/d87bdd7c2f1bffb2552ed6443d6409bf.jpg\",\n", + " \"teacher\": \"冯老师\",\n", + " \"Photonames\": \"1612857034.mp4\",\n", + " \"url\": \"http://www.tongzhuo100.com/v/2021-02/158070.html\",\n", + " \"checkfree\": \"0\",\n", + " \"playtime\": \"00:22:35\",\n", + " \"svisited\": \"1000000\",\n", + " \"sequens\": \"53\",\n", + " \"sdate\": \"1612857090\"\n", + " }\n", + " ]\n", + " },\n", + " \"error\": \"\"\n", + "}\n", + "'''\n", + "rows = json.loads(json_str)['responsedata']['rows']\n", + "#print (data2['data']['playarray'])\n", + "for row in rows:\n", + " for value in row:\n", + " print(row[value])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/.ipynb_checkpoints/微信公众号文章爬取-checkpoint.ipynb b/.ipynb_checkpoints/微信公众号文章爬取-checkpoint.ipynb new file mode 100644 index 0000000..e59b0f9 --- /dev/null +++ b/.ipynb_checkpoints/微信公众号文章爬取-checkpoint.ipynb @@ -0,0 +1,438 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 70, + "metadata": {}, + "outputs": [], + "source": [ + "# -*- coding:utf-8 -*-\n", + "import requests\n", + "import time\n", + "import csv\n", + "import json\n", + "from lxml import etree " + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "metadata": {}, + "outputs": [], + "source": [ + "url=\"https://sd.dzwww.com/sdnews/default.htm\"\n", + "header={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'}" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "metadata": {}, + "outputs": [], + "source": [ + "res=requests.get(url,headers=header)\n", + "html=etree.HTML(res.text.encode(\"latin1\").decode(\"gbk\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 73, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "50\n", + "['盘点2019展望2020|打造样板，乡村振兴全面起势', '攻坚2020|攻坚关键在项目，山东各地抓项目如何“心中有数”“手中有招”？', '将原油码头“搬”到炼化企业门口 连接黄渤海“大油龙”全线贯通', '57个果品品牌获评齐鲁放心果品 葡萄拔得头筹苹果紧随其后', '“45度让路法”风靡网络 路遇特种车辆，如何礼让', '济南网约车细则再征民意：2月1日起施行，车辆需安装定位', '对有偿补课零容忍！济南通报12起典型问题', '双招双引，看如何招得好留得住？大众日报整版经验来了', '国务院批复 青岛再添国家级功能区 直接选中这里', '济南:将项目资金分配权交给部门 把项目确定权交给基层', '不到一月，三次半路停摆！ 唐骏箱货新车“带病”出厂就“添堵”', '问政山东丨携号转网靓号不给转还乱加业务包？省通信局长：有人为设置障碍', '问政山东丨“非实名”电话卡仍在售 店主打包票“只要不违法就不会被封”', '春运期间，加开7列途经潍坊站的列车', '济南网约车门槛“有降有升”，高德等聚合平台也要承担审核责任', '青岛纺织服装行业如何互联网升级 专业人士给方案', '2020春运大幕将拉开 青岛铁路预计发送240万人次', '“天价高速费”最新进展：山西方面退还1100元，并道歉！', '2020年将有这些新大学在山东落地 青岛成为主要建校地？', '自贸区青岛片区首个平台型经济落户西海岸新区', '世界级创新中心落户高新区 五年将聚集300家企业!', '定了！济南泉城路真正变身步行街，非驻区车辆将绕行外围', '自来水管道破损致济南经十路“成河”，他们泡水里抢修一夜没合眼', '“开门红”!2020年青岛首批专项债券80亿元顺利发行', '烟台芝罘大海阳过街天桥主体将于春节前完工，预计3月底前投用', '推动市区一体融合发展，烟台全长24公里的跨区输水管网正在启动', '济南一特色小镇项目规划有进展，将形成大规模办公区', '注意啦！济南二环北路药山西路交叉口施工，K127路临时调线', '14岁女孩长期排便困难就要带着“粪兜”过活？八旬老教授say no！', '山东首个！济南西站开通滴滴网约车专属通道', '有一种冷叫妈妈觉得你冷！孩子冬天穿太多反而更易感冒', '2020年山东全面放宽落户条件 这3个市要建成垃圾分类处理系统', '女子投诉快递员不送件竟遭登门暴打！韵达快递：已开除涉事员工', '问政山东丨两年拉黑了1000多个号码 骚扰电话“阴魂不散”该咋办？', '“雪窝”烟台清雪防滑有高招！巧用环保型融雪剂保通畅', '预防流感守护健康 医生提醒：幼儿、老人、孕妇需格外注意', '东营：雪夜一男子落水 4名民警“搭人梯”救人', '退役军人事务工作表彰奖励办法出炉 有这3个常设项目', '寻找青年“政治佳”第四季获奖作品抢先看！', '一年出警586万人次！2019年济南实现街面“两抢”315天零发案', '山财原党委副书记车滨已出任德州学院党委书记', '济莱高铁全线完成招标 钢城区段已具备施工条件', '村民称“俺村进狼了偷鸡吃” 民警全副武装赶到结果……', '手机用户呈低龄化，低劣内容借移动社交隐蔽传播', '@2020年高考生！山东新高考及模拟工作权威解析来了', '淄博：隆冬季节 消防员趴在地上救助被压车底老人', '早上入院治疗晚上就能出院 山东首家省级日间病房开始运行', '山东：推动开发区体制机制改革 打造高质量发展示范区', '村民称“俺村进狼了偷鸡吃” 民警全副武装赶到结果……', '车祸受害者治病花了30多万陷困境，加害人又无力赔偿怎么办？']\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4641108.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4641186.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4641111.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4641113.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4641112.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4641320.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4641277.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4641268.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4641259.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4641330.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4688310.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4688303.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4688271.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4688253.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4688251.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4688236.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4688209.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4688208.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4688207.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4688204.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4688194.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4688183.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4688180.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4688179.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4688175.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4688166.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4688161.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4688158.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4688057.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4688042.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4688014.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4688003.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4687984.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4687966.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4687911.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4687892.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4687870.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4687834.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4687823.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4687816.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4687803.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4687802.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4687773.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4687736.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4687735.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4687732.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4687654.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4687605.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4687592.htm\n", + "https://sd.dzwww.com/sdnews/202001/d20200109_4687587.htm\n" + ] + } + ], + "source": [ + "#获取新闻标题\n", + "news_titles=[]\n", + "news_a=html.xpath('//div[3]/ul/li/h3/a')\n", + "print(len(news_a))\n", + "for a in news_a:\n", + " news_titles.append(a.xpath('string(.)').strip())\n", + "print(news_titles)\n", + "#获取新闻链接\n", + "news_links=html.xpath('//div[3]/ul/li/h3/a/@href')\n", + "for link in news_links:\n", + " print(link)" + ] + }, + { + "cell_type": "code", + "execution_count": 95, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "盘点2019展望2020|打造样板，乡村振兴全面起势\n", + "2020 01/09 05:45\n", + "来源：大众日报\n", + "作者：毛鑫鑫 张 鹏 于新悦\n", + "攻坚2020|攻坚关键在项目，山东各地抓项目如何“心中有数”“手中有招”？\n", + "2020 01/09 06:17\n", + "来源：大众日报客户端\n", + "作者：禹亚宁 孙秀岭\n", + "将原油码头“搬”到炼化企业门口 连接黄渤海“大油龙”全线贯通\n", + "2020 01/09 05:47\n", + "来源：大众日报\n", + "作者：李媛　王晶\n", + "57个果品品牌获评齐鲁放心果品 葡萄拔得头筹苹果紧随其后\n", + "2020 01/09 05:50\n", + "来源：大众日报\n", + "作者：\n", + "“45度让路法”风靡网络 路遇特种车辆，如何礼让\n", + "2020 01/09 05:50\n", + "来源：大众日报\n", + "作者：马海燕　张依盟\n", + "济南网约车细则再征民意：2月1日起施行，车辆需安装定位\n", + "2020 01/09 06:43\n", + "来源：济南时报\n", + "作者：\n", + "对有偿补课零容忍！济南通报12起典型问题\n", + "2020 01/09 06:38\n", + "来源：济南日报\n", + "作者：\n", + "双招双引，看如何招得好留得住？大众日报整版经验来了\n", + "2020 01/09 06:37\n", + "来源：大众日报\n", + "作者：张晓帆 吕光社 杨国胜 张 蓓\n", + "国务院批复 青岛再添国家级功能区 直接选中这里\n", + "2020 01/09 06:36\n", + "来源：青岛新闻网\n", + "作者：\n", + "济南:将项目资金分配权交给部门 把项目确定权交给基层\n", + "2020 01/09 06:45\n", + "来源：济南日报\n", + "作者：\n", + "不到一月，三次半路停摆！ 唐骏箱货新车“带病”出厂就“添堵”\n", + "2020 01/09 21:30\n", + "来源：大众网·海报新闻\n", + "作者：石磊\n", + "问政山东丨携号转网靓号不给转还乱加业务包？省通信局长：有人为设置障碍\n", + "2020 01/09 21:29\n", + "来源：大众网·海报新闻\n", + "作者：辛振东\n", + "问政山东丨“非实名”电话卡仍在售 店主打包票“只要不违法就不会被封”\n", + "2020 01/09 21:17\n", + "来源：大众网·海报新闻\n", + "作者：张稳\n", + "春运期间，加开7列途经潍坊站的列车\n", + "2020 01/09 21:11\n", + "来源：大众日报客户端\n", + "作者：大众日报客户端记者 张蓓\n", + "济南网约车门槛“有降有升”，高德等聚合平台也要承担审核责任\n", + "2020 01/09 21:10\n", + "来源：齐鲁晚报·齐鲁壹点\n", + "作者：刘飞跃\n", + "青岛纺织服装行业如何互联网升级 专业人士给方案\n", + "2020 01/09 21:06\n", + "来源：青岛新闻网\n", + "作者：刘倩倩\n", + "2020春运大幕将拉开 青岛铁路预计发送240万人次\n", + "2020 01/09 21:02\n", + "来源：青岛新闻网\n", + "作者：江东旭\n", + "“天价高速费”最新进展：山西方面退还1100元，并道歉！\n", + "2020 01/09 21:01\n", + "来源：大众网·海报新闻\n", + "作者：解强民\n", + "2020年将有这些新大学在山东落地 青岛成为主要建校地？\n", + "2020 01/09 21:01\n", + "来源：青岛新闻网\n", + "作者：\n", + "自贸区青岛片区首个平台型经济落户西海岸新区\n", + "2020 01/09 21:01\n", + "来源：青岛新闻网\n", + "作者：陈志伟\n", + "世界级创新中心落户高新区 五年将聚集300家企业!\n", + "2020 01/09 21:00\n", + "来源：青岛新闻网\n", + "作者：任俊峰\n", + "定了！济南泉城路真正变身步行街，非驻区车辆将绕行外围\n", + "2020 01/09 20:59\n", + "来源：齐鲁晚报·齐鲁壹点\n", + "作者：张泰来 杜亚慧\n", + "自来水管道破损致济南经十路“成河”，他们泡水里抢修一夜没合眼\n", + "2020 01/09 20:59\n", + "来源：齐鲁晚报·齐鲁壹点\n", + "作者：程凌润 孙雨杨\n", + "“开门红”!2020年青岛首批专项债券80亿元顺利发行\n", + "2020 01/09 20:58\n", + "来源：青岛新闻网\n", + "作者：刘倩倩\n", + "烟台芝罘大海阳过街天桥主体将于春节前完工，预计3月底前投用\n", + "2020 01/09 20:58\n", + "来源：齐鲁晚报·齐鲁壹点\n", + "作者：张菁\n", + "推动市区一体融合发展，烟台全长24公里的跨区输水管网正在启动\n", + "2020 01/09 20:57\n", + "来源：齐鲁晚报·齐鲁壹点\n", + "作者：曲彦霖\n", + "济南一特色小镇项目规划有进展，将形成大规模办公区\n", + "2020 01/09 20:57\n", + "来源：齐鲁晚报·齐鲁壹点\n", + "作者：吕琳\n", + "注意啦！济南二环北路药山西路交叉口施工，K127路临时调线\n", + "2020 01/09 20:57\n", + "来源：齐鲁晚报·齐鲁壹点\n", + "作者：刘飞跃\n", + "14岁女孩长期排便困难就要带着“粪兜”过活？八旬老教授say no！\n", + "2020 01/09 20:38\n", + "来源：大众网·海报新闻\n", + "作者：董昊骞\n", + "山东首个！济南西站开通滴滴网约车专属通道\n", + "2020 01/09 20:35\n", + "来源：大众网·海报新闻\n", + "作者：宋冰\n", + "有一种冷叫妈妈觉得你冷！孩子冬天穿太多反而更易感冒\n", + "2020 01/09 20:31\n", + "来源：大众网·海报新闻\n", + "作者：董昊骞\n", + "2020年山东全面放宽落户条件 这3个市要建成垃圾分类处理系统\n", + "2020 01/09 20:28\n", + "来源：大众网·海报新闻\n", + "作者：张稳 吕乐\n", + "女子投诉快递员不送件竟遭登门暴打！韵达快递：已开除涉事员工\n", + "2020 01/09 20:20\n", + "来源：大众网·海报新闻\n", + "作者：\n", + "问政山东丨两年拉黑了1000多个号码 骚扰电话“阴魂不散”该咋办？\n", + "2020 01/09 20:13\n", + "来源：大众网·海报新闻\n", + "作者：张稳\n", + "“雪窝”烟台清雪防滑有高招！巧用环保型融雪剂保通畅\n", + "2020 01/09 20:05\n", + "来源：大众网·海报新闻\n", + "作者：杨瑞远\n", + "预防流感守护健康 医生提醒：幼儿、老人、孕妇需格外注意\n", + "2020 01/09 20:04\n", + "来源：大众网·海报新闻\n", + "作者：吴宝杰\n", + "东营：雪夜一男子落水 4名民警“搭人梯”救人\n", + "2020 01/09 20:02\n", + "来源：大众网·海报新闻\n", + "作者：陈丽伟\n", + "退役军人事务工作表彰奖励办法出炉 有这3个常设项目\n", + "2020 01/09 20:00\n", + "来源：大众网·海报新闻\n", + "作者：秦文\n", + "寻找青年“政治佳”第四季获奖作品抢先看！\n", + "2020 01/09 19:57\n", + "来源：大众网·海报新闻\n", + "作者：\n", + "一年出警586万人次！2019年济南实现街面“两抢”315天零发案\n", + "2020 01/09 19:55\n", + "来源：大众网·海报新闻\n", + "作者：张珈玮\n", + "山财原党委副书记车滨已出任德州学院党委书记\n", + "2020 01/09 19:51\n", + "来源：大众网·海报新闻\n", + "作者：孙杰\n", + "济莱高铁全线完成招标 钢城区段已具备施工条件\n", + "2020 01/09 19:50\n", + "来源：大众网·海报新闻\n", + "作者：解强民\n", + "村民称“俺村进狼了偷鸡吃” 民警全副武装赶到结果……\n", + "2020 01/09 19:48\n", + "来源：大众网·海报新闻\n", + "作者：解强民\n", + "手机用户呈低龄化，低劣内容借移动社交隐蔽传播\n", + "2020 01/09 19:46\n", + "来源：新时报\n", + "作者：\n", + "@2020年高考生！山东新高考及模拟工作权威解析来了\n", + "2020 01/09 19:46\n", + "来源：大众日报客户端\n", + "作者：大众日报客户端记者 王桂利\n", + "淄博：隆冬季节 消防员趴在地上救助被压车底老人\n", + "2020 01/09 19:45\n", + "来源：大众日报客户端\n", + "作者：大众日报记者 张依盟\n", + "早上入院治疗晚上就能出院 山东首家省级日间病房开始运行\n", + "2020 01/09 19:43\n", + "来源：大众网·海报新闻\n", + "作者：董昊骞\n", + "山东：推动开发区体制机制改革 打造高质量发展示范区\n", + "2020 01/09 19:35\n", + "来源：齐鲁网\n", + "作者：\n", + "村民称“俺村进狼了偷鸡吃” 民警全副武装赶到结果……\n", + "2020 01/09 19:33\n", + "来源：大众网·海报新闻\n", + "作者：\n", + "车祸受害者治病花了30多万陷困境，加害人又无力赔偿怎么办？\n", + "2020 01/09 19:30\n", + "来源：齐鲁晚报·齐鲁壹点\n", + "作者：杨璐\n" + ] + } + ], + "source": [ + "#爬取文章内容\n", + "import re\n", + "content=[]\n", + "for news_link in news_links:\n", + " c_res = requests.get(news_link,headers=header)\n", + " c_html = etree.HTML(c_res.text.encode(\"latin1\").decode(\"gbk\"))\n", + " s = etree.tostring(c_html).decode()\n", + " #print(s)\n", + " p = c_html.xpath('//div[@id=\"news-body\"]')\n", + " c_body = p[0].xpath('string(.)').strip()\n", + " c_body = re.sub('<[^<]+?>', '', c_body).replace('\\n', '').strip()\n", + " c_body = re.sub(r'', '', c_body)\n", + " #print(c_body)\n", + " p = c_html.xpath('//div[@id=\"news-head\"]/h2')\n", + " c_title = p[0].xpath('string(.)').strip()\n", + " print(c_title)\n", + " p = c_html.xpath('//div[@id=\"news-side\"]/div[1]')\n", + " c_time = p[0].xpath('string(.)').strip()\n", + " c_time = c_time.replace('\\n',' ')\n", + " c_time = c_time.replace('\\t','')\n", + " print(c_time)\n", + " p = c_html.xpath('//div[@id=\"news-side\"]/div[2]')\n", + " c_laiyuan = p[0].xpath('string(.)').strip()\n", + " c_laiyuan = c_laiyuan.replace('\\n','')\n", + " c_laiyuan = c_laiyuan.replace('\\t','')\n", + " print(c_laiyuan)\n", + " p = c_html.xpath('//div[@id=\"news-side\"]/div[3]')\n", + " c_zuozhe = p[0].xpath('string(.)').strip()\n", + " c_zuozhe = c_zuozhe.replace('\\n','')\n", + " c_zuozhe = c_zuozhe.replace('\\t','')\n", + " print(c_zuozhe)" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "ename": "FileNotFoundError", + "evalue": "[Errno 2] No such file or directory: 'data/weixin_zhonyin.csv'", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[1;31m#实现多页爬取和存入csv\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[1;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"data/weixin_zhonyin.csv\"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'a+'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mnewline\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m''\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mencoding\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'utf-8'\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mfile\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 3\u001b[0m \u001b[0mcsv_writer\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcsv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwriter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 4\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 5\u001b[0m \u001b[0mtitles\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'data/weixin_zhonyin.csv'" + ] + } + ], + "source": [ + "#实现多页爬取和存入csv\n", + "with open(\"data/weixin_zhonyin.csv\",'a+',newline='',encoding='utf-8') as file:\n", + " csv_writer=csv.writer(file)\n", + " for i in range(2):\n", + " titles=[]\n", + " contents=[]\n", + " url=\"http://weixin.sogou.com/weixin?oq=&query={}&_sug_type_=1&sut=0&lkt=0%2C0%2C0&s_from=input&ri=1&_sug_=n&type=2&sst0=1509458456759&page={}&ie=utf8&p=40040108&dp=1&w=01015002&dr=1\".format(\"中印对峙\",i+1)\n", + " res=requests.get(url,headers=header)\n", + " html=etree.HTML(res.text)\n", + " news_a=html.xpath('//div[@class=\"news-box\"]/ul[@class=\"news-list\"]/li/div[@class=\"txt-box\"]/h3/a')\n", + " for a in news_a:\n", + " titles.append(a.xpath('string(.)').strip())\n", + "# print(titles)\n", + " links=html.xpath('//div[@class=\"news-box\"]/ul[@class=\"news-list\"]/li/div[@class=\"txt-box\"]/h3/a/@href')\n", + " for link in links:\n", + " c_res=requests.get(link,headers=header)\n", + " c_html=etree.HTML(c_res.text)\n", + " p=c_html.xpath('//div[@id=\"js_content\"]/p/text()')\n", + " if p:\n", + " contents.append('。'.join(p))\n", + " else:\n", + " p=c_html.xpath('//div[@id=\"js_content\"]/p/span/text()')\n", + " contents.append('。'.join(p))\n", + " for data in zip(titles,links,contents):\n", + " print(data)\n", + " csv_writer.writerow(list(data))\n", + " time.sleep(2)\n", + " print(\"第{}页爬取完毕\".format(i+1))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.4" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/.ipynb_checkpoints/抖音测试用-checkpoint.ipynb b/.ipynb_checkpoints/抖音测试用-checkpoint.ipynb new file mode 100644 index 0000000..b180e9c --- /dev/null +++ b/.ipynb_checkpoints/抖音测试用-checkpoint.ipynb @@ -0,0 +1,205 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "手机型号：ONEPLUS A5010\n" + ] + } + ], + "source": [ + "import time\n", + "import uiautomator2 as u2\n", + "\n", + "\n", + "d = u2.connect_wifi('192.168.10.136')\n", + "\n", + "print('手机型号：'+d.device_info['model'])\n", + "## 我\n", + "my = d.xpath('//*[@text=\"我\"]')\n", + "my = d.xpath('//*[@resource-id=\"com.ss.android.ugc.aweme:id/root_view\"]/android.widget.FrameLayout[5]/android.widget.RelativeLayout[1]/android.widget.RelativeLayout[1]')\n", + "my.click()\n", + "## 判断是否出现 编辑资料 出现则表示在我的页面\n", + "d.xpath('//*[@resource-id=\"com.ss.android.ugc.aweme:id/ht7\"]')\n", + "\n", + "## 我的抖音号\n", + "d.xpath('//*[@resource-id=\"com.ss.android.ugc.aweme:id/l1c\"]')\n", + "## 获赞\n", + "d.xpath('//*[@resource-id=\"com.ss.android.ugc.aweme:id/b4o\"]')\n", + "## 粉丝数\n", + "d.xpath('//*[@resource-id=\"com.ss.android.ugc.aweme:id/c__\"]')\n", + "## 我的关注\n", + "d.xpath('//*[@resource-id=\"com.ss.android.ugc.aweme:id/c-d\"]')\n", + "\n", + "## 右上角打开侧边栏按钮\n", + "info = d.xpath('//*[@resource-id=\"com.ss.android.ugc.aweme:id/ghv\"]')\n", + "info.click()\n", + "## 点击侧边栏的创作中心\n", + "d.xpath('//*[@text=\"创作者服务中心\"]').click()\n", + "\n", + "d.xpath('//*[@text=\"商品橱窗\"]').click()\n", + "\n", + "## 这个是广告 后面处理\n", + "d.xpath('//*[@text=\"稍后再看\"]').click()\n", + "\n", + "d.xpath('//*[@text=\"选品广场\"]').click()\n", + "\n", + "d.xpath('//*[@text=\"服饰内衣\"]')\n", + "## 通过右上角 搜索后面的链接判断是否进入页面 此处应该等于 链接\n", + "d.xpath('//*[@resource-id=\"app\"]/android.view.View[1]/android.view.View[1]/android.view.View[3]')\n", + "d.xpath('//*[@text=\"链接\"]')\n", + "\n", + "## 点击子分类进入商品页\n", + "d.xpath('//*[@text=\"裤子\"]')\n", + "## 可以用于判断商品列表页面是否打开\n", + "text = d.xpath('//*[@resource-id=\"app\"]/android.view.View[1]/android.view.View[1]/android.view.View[2]')\n", + "d.xpath('//*[@text=\"推荐\"]')\n", + "## 这个地方是推荐就代表打开了商品列表页面\n", + "d.xpath('//*[@resource-id=\"app\"]/android.view.View[1]/android.view.View[2]/android.view.View[1]')\n", + "## 第一个是筛选栏 所以要-1\n", + "good_num = d.xpath('//*[@resource-id=\"app\"]/android.view.View').all()\n", + "## 判断是否打开了商品详情页 如果打开页面标题应该为 商品推广信息\n", + "d.xpath('//*[@resource-id=\"com.ss.android.ugc.aweme:id/title\"]')\n", + "## 售价\n", + "m_money = d.xpath('//*[@resource-id=\"app\"]/android.view.View[1]/android.view.View[1]/android.view.View[2]/android.view.View[3]/android.view.View[3]')\n", + "store_name = d.xpath('//*[@resource-id=\"app\"]/android.view.View[1]/android.view.View[1]/android.view.View[14]/android.view.View[1]/android.view.View[1]/android.view.View[2]/android.view.View[1]')\n", + "## 预计每当赚 后面三个分别为 百十个位 小数点 小数位\n", + "lirun = d.xpath('//*[@resource-id=\"app\"]/android.view.View[1]/android.view.View[1]/android.view.View[2]/android.view.View[1]/android.view.View[3]')\n", + "## 点击联系客服\n", + "d.xpath('//*[@text=\"联系商家\"]')\n", + "## 判断是否进入客服页面\n", + "d.xpath('//*[@resource-id=\"web-im-container\"]')\n", + "## 电话按钮\n", + "d.xpath('//*[@resource-id=\"web-im-container\"]/android.view.View[1]/android.view.View[1]/android.view.View[1]/android.view.View[1]/android.view.View[2]/android.view.View[4]/android.view.View[1]')\n", + "## 出现联系商家\n", + "d.xpath('//*[@text=\"联系商家\"]')\n", + "## 这里是电话沟通 还是 查看商家联系方式\n", + "d.xpath('//android.widget.ListView/android.view.View[1]/android.view.View[2]')\n", + "## 如果是联系方式 则点击查看\n", + "d.xpath('//*[@text=\"查看商家联系方式\"]')\n", + "## 点击 复制号码\n", + "d.xpath('//*[@text=\"复制号码\"]')\n", + "## 通过剪贴板获取\n", + "print(d.clipboard)\n", + "## 可以清空剪贴板\n", + "d.set_clipboard('text', 'label')\n", + "\n", + "## 点击弹出联系方式的取消\n", + "d.xpath('//*[@text=\"取消\"]')\n", + "## 返回两次到达商品列表页\n", + "d.press(\"back\")\n", + "d.press(\"back\")\n", + "\n", + "all_type = d.xpath('//*[@resource-id=\"app\"]/android.view.View[1]/android.view.View[2]/android.view.View[1]/android.view.View[1]/android.view.View[2]')\n", + "all_type.click()\n", + "\n", + "one_type = d.xpath('//*[@text=\"食品饮料\"]')\n", + "## 滑动侧栏\n", + "\n", + "\n", + "d.HTTP_TIMEOUT = 60 # 默认值60s, http默认请求超时时间\n", + "\n", + "# 当设备掉线时，等待设备在线时长，仅当TMQ=true时有效，支持通过环境变量 WAIT_FOR_DEVICE_TIMEOUT 设置\n", + "d.WAIT_FOR_DEVICE_TIMEOUT = 70 \n", + "\n", + "d(text=\"Settings\").exists # True if exists, else False\n", + "d.exists(text=\"Settings\") # alias of above property.\n", + "\n", + "# advanced usage\n", + "d(text=\"Settings\").exists(timeout=3) # wait Settings appear in 3s, same as .wait(3)\n", + "\n", + "d(text=\"Settings\").get_text() # get widget text\n", + "d(text=\"Settings\").set_text(\"My text...\") # set the text\n", + "d(text=\"Settings\").clear_text() # clear the text\n", + "\n", + "## Swipe from the center of the UI object to its edge\n", + "\n", + "## Swipe supports 4 directions:\n", + "## left\n", + "## right\n", + "## top\n", + "## bottom\n", + "d(text=\"Settings\").swipe(\"right\")\n", + "d(text=\"Settings\").swipe(\"left\", steps=10)\n", + "d(text=\"Settings\").swipe(\"up\", steps=20) # 1 steps is about 5ms, so 20 steps is about 0.1s\n", + "d(text=\"Settings\").swipe(\"down\", steps=20)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# coding: utf-8\n", + "#\n", + "import uiautomator2 as u2\n", + "from uiautomator2 import Direction\n", + "\n", + "d = u2.connect('192.168.10.136')\n", + "#d.debug = True\n", + "print(d.app_current())\n", + "print(d.device_info)\n", + "#d.click(0.525, 0.151)\n", + "#d.swipe_ext(\"up\", scale=0.9)\n", + "#d.app_list_running()\n", + "#d(text=\"Settings\").swipe(\"down\", steps=20)\n", + "\n", + "stores = d.xpath('//*[@resource-id=\"app\"]/android.view.View').all()\n", + "#store_count = count(stores)\n", + "#print(store_count)\n", + "#d.swipe_points([(200,1000),(250,500),(250,100)],0.2)\n", + "\n", + "\n", + "for store in stores:\n", + " good_attrib = store.elem.find('./android.view.View[2]')\n", + " good_name = good_attrib.attrib['text']\n", + " print(good_name)\n", + " if good_name != '':\n", + " store.click()\n", + " \n", + " title = d.xpath('//*[@resource-id=\"com.ss.android.ugc.aweme:id/title\"]').get_text()\n", + " z_money1 = d.xpath('//*[@text=\"预估每单赚\"]').parent().elem.find('./android.view.View[3]').attrib['text']\n", + " z_money2 = d.xpath('//*[@text=\"预估每单赚\"]').parent().elem.find('./android.view.View[5]').attrib['text']\n", + " z_money = z_money1 + '.' + z_money2\n", + " m_money = d.xpath('//*[@text=\"售价\"]').parent().elem.find('./android.view.View[3]').attrib['text']\n", + " d.xpath('//*[@resource-id=\"app\"]/android.view.View[1]').scroll_to(\"进店逛逛\")\n", + " store_name = d.xpath('//*[@text=\"进店逛逛\"]').parent().elem.find('./android.view.View[1]/android.view.View[2]').attrib['text']\n", + " print(z_money,m_money,store_name)\n", + " d(resourceId=\"com.ss.android.ugc.aweme:id/bbf\").click()\n", + " # 按下back键的常规方式\n", + " #d.swipe_ext(\"down\")\n", + "\n", + "#print(d(resourceId=\"//*[@resource-id=\"wall-warper\"]/android.view.View[2]/android.view.View[1]/android.widget.Button\").count)\n", + "#d.send_keys(\"back\", clear=True)\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:root] *", + "language": "python", + "name": "conda-root-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/.ipynb_checkpoints/测试-checkpoint.ipynb b/.ipynb_checkpoints/测试-checkpoint.ipynb new file mode 100644 index 0000000..30ac1c3 --- /dev/null +++ b/.ipynb_checkpoints/测试-checkpoint.ipynb @@ -0,0 +1,84 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "https://v92-dy.ixigua.com/65e0f20a39ff8801b1a75cb13772f885/5f9b1f3e/video/tos/cn/tos-cn-ve-15/05235eb3a2bd4bf4b39c3e613312edb6/?a=1128&br=2649&bt=883&cr=0&cs=0&cv=1&dr=0&ds=6&er=&l=202010300259290101980670282B3A923F&lr=&mime_type=video_mp4&qs=0&rc=M3hkZjhmMzM7djMzN2kzM0ApaDxoZ2ZoNjs5Nzg3NzY5PGdgMzJhcGVvczFfLS0zLS9zczUyNTNgNjEuYTJhYzA2LjA6Yw%3D%3D&vl=&vr=\n" + ] + } + ], + "source": [ + "import re\n", + "\n", + "import requests\n", + "\n", + "headers = {\n", + " 'User-Agent': 'User-Agent\", \"Mozilla/5.0 (iPhone; CPU iPhone OS 12_1_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/16D57 Version/12.0 Safari/604.1',\n", + "}\n", + "\n", + "url = 'https://v.douyin.com/J8X86mA/' # 抖音分享的链接，自行更改\n", + "response = requests.get(url, headers=headers)\n", + "\n", + "now_url = response.url\n", + "pat_item_ids = '/video/(.*?)/'\n", + "item_ids = re.compile(pat_item_ids, re.S).findall(now_url)\n", + "\n", + "pat_dytk = 'dytk: \"(.*?)\"'\n", + "dytk = re.compile(pat_dytk, re.S).findall(response.text)\n", + "\n", + "url = 'https://www.iesdouyin.com/web/api/v2/aweme/iteminfo/'\n", + "params = {\n", + " 'item_ids': item_ids,\n", + " 'dytk': dytk\n", + "}\n", + "response = requests.get(url, headers=headers, params=params).json()\n", + "true_url = response['item_list'][0]['video']['play_addr']['url_list'][0]\n", + "true_url = true_url.replace('playwm','play')\n", + "response = requests.get(true_url, headers=headers)\n", + "true_url = response.url\n", + "print(true_url)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/.ipynb_checkpoints/阿里云DDNS-checkpoint.ipynb b/.ipynb_checkpoints/阿里云DDNS-checkpoint.ipynb new file mode 100644 index 0000000..1f5020f --- /dev/null +++ b/.ipynb_checkpoints/阿里云DDNS-checkpoint.ipynb @@ -0,0 +1,131 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "75e8667f-b8d9-4b1d-a119-9babdfc24947", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Looking in indexes: https://mirrors.aliyun.com/pypi/simple/, https://pypi.ngc.nvidia.com\n", + "Collecting aliyun-python-sdk-core\n", + " Downloading https://mirrors.aliyun.com/pypi/packages/1e/e3/8623c0305022610466ded2b0010a7221e9585046116263dd27cb2e56df36/aliyun-python-sdk-core-2.14.0.tar.gz (443 kB)\n", + " -------------------------------------- 443.0/443.0 kB 5.5 MB/s eta 0:00:00\n", + " Preparing metadata (setup.py): started\n", + " Preparing metadata (setup.py): finished with status 'done'\n", + "Collecting jmespath<1.0.0,>=0.9.3\n", + " Downloading https://mirrors.aliyun.com/pypi/packages/07/cb/5f001272b6faeb23c1c9e0acc04d48eaaf5c862c17709d20e3469c6e0139/jmespath-0.10.0-py2.py3-none-any.whl (24 kB)\n", + "Requirement already satisfied: cryptography>=2.6.0 in d:\\miniconda3\\lib\\site-packages (from aliyun-python-sdk-core) (41.0.7)\n", + "Requirement already satisfied: cffi>=1.12 in d:\\miniconda3\\lib\\site-packages (from cryptography>=2.6.0->aliyun-python-sdk-core) (1.16.0)\n", + "Requirement already satisfied: pycparser in d:\\miniconda3\\lib\\site-packages (from cffi>=1.12->cryptography>=2.6.0->aliyun-python-sdk-core) (2.21)\n", + "Building wheels for collected packages: aliyun-python-sdk-core\n", + " Building wheel for aliyun-python-sdk-core (setup.py): started\n", + " Building wheel for aliyun-python-sdk-core (setup.py): finished with status 'done'\n", + " Created wheel for aliyun-python-sdk-core: filename=aliyun_python_sdk_core-2.14.0-py3-none-any.whl size=535295 sha256=319f4d1ebfbba997c6937d76897cfabf74107f17cf39443b590782bb0767cd05\n", + " Stored in directory: C:\\Users\\meng\\AppData\\Local\\Temp\\pip-ephem-wheel-cache-1m9gv2st\\wheels\\b7\\c5\\b1\\1864d88c500f3e74759e9a607a3d2274d70b9409337da7df81\n", + "Successfully built aliyun-python-sdk-core\n", + "Installing collected packages: jmespath, aliyun-python-sdk-core\n", + "Successfully installed aliyun-python-sdk-core-2.14.0 jmespath-0.10.0\n", + "Looking in indexes: https://mirrors.aliyun.com/pypi/simple/, https://pypi.ngc.nvidia.com\n", + "Collecting aliyun-python-sdk-domain\n", + " Downloading https://mirrors.aliyun.com/pypi/packages/5b/4e/1c7026804b86a470d890f1cf1d5f0f20f6de9521cab7288d935c6d32a302/aliyun_python_sdk_domain-3.14.9-py2.py3-none-any.whl (134 kB)\n", + " -------------------------------------- 134.6/134.6 kB 4.0 MB/s eta 0:00:00\n", + "Requirement already satisfied: aliyun-python-sdk-core>=2.11.5 in d:\\miniconda3\\lib\\site-packages (from aliyun-python-sdk-domain) (2.14.0)\n", + "Requirement already satisfied: cryptography>=2.6.0 in d:\\miniconda3\\lib\\site-packages (from aliyun-python-sdk-core>=2.11.5->aliyun-python-sdk-domain) (41.0.7)\n", + "Requirement already satisfied: jmespath<1.0.0,>=0.9.3 in d:\\miniconda3\\lib\\site-packages (from aliyun-python-sdk-core>=2.11.5->aliyun-python-sdk-domain) (0.10.0)\n", + "Requirement already satisfied: cffi>=1.12 in d:\\miniconda3\\lib\\site-packages (from cryptography>=2.6.0->aliyun-python-sdk-core>=2.11.5->aliyun-python-sdk-domain) (1.16.0)\n", + "Requirement already satisfied: pycparser in d:\\miniconda3\\lib\\site-packages (from cffi>=1.12->cryptography>=2.6.0->aliyun-python-sdk-core>=2.11.5->aliyun-python-sdk-domain) (2.21)\n", + "Installing collected packages: aliyun-python-sdk-domain\n", + "Successfully installed aliyun-python-sdk-domain-3.14.9\n", + "Looking in indexes: https://mirrors.aliyun.com/pypi/simple/, https://pypi.ngc.nvidia.com\n", + "Collecting aliyun-python-sdk-alidns\n", + " Downloading https://mirrors.aliyun.com/pypi/packages/71/f4/2f70945b351cbffae37db518ba46f2936cc22d4d4d3969db7f4b5aac0d21/aliyun_python_sdk_alidns-3.0.7-py2.py3-none-any.whl (177 kB)\n", + " -------------------------------------- 177.1/177.1 kB 5.2 MB/s eta 0:00:00\n", + "Requirement already satisfied: aliyun-python-sdk-core>=2.11.5 in d:\\miniconda3\\lib\\site-packages (from aliyun-python-sdk-alidns) (2.14.0)\n", + "Requirement already satisfied: cryptography>=2.6.0 in d:\\miniconda3\\lib\\site-packages (from aliyun-python-sdk-core>=2.11.5->aliyun-python-sdk-alidns) (41.0.7)\n", + "Requirement already satisfied: jmespath<1.0.0,>=0.9.3 in d:\\miniconda3\\lib\\site-packages (from aliyun-python-sdk-core>=2.11.5->aliyun-python-sdk-alidns) (0.10.0)\n", + "Requirement already satisfied: cffi>=1.12 in d:\\miniconda3\\lib\\site-packages (from cryptography>=2.6.0->aliyun-python-sdk-core>=2.11.5->aliyun-python-sdk-alidns) (1.16.0)\n", + "Requirement already satisfied: pycparser in d:\\miniconda3\\lib\\site-packages (from cffi>=1.12->cryptography>=2.6.0->aliyun-python-sdk-core>=2.11.5->aliyun-python-sdk-alidns) (2.21)\n", + "Installing collected packages: aliyun-python-sdk-alidns\n", + "Successfully installed aliyun-python-sdk-alidns-3.0.7\n" + ] + } + ], + "source": [ + "!pip3 install aliyun-python-sdk-core\n", + "!pip3 install aliyun-python-sdk-domain\n", + "!pip3 install aliyun-python-sdk-alidns" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "2b5c4f80-5b91-4366-814b-26af5618c099", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "域名列表 {\"Domains\":{\"Domain\":[]},\"TotalCount\":0,\"PageSize\":20,\"RequestId\":\"1C6E952B-2DAB-538E-B854-85B82203759B\",\"PageNumber\":1}\n" + ] + } + ], + "source": [ + "# 加载核心SDK\n", + "from aliyunsdkcore.client import AcsClient\n", + "from aliyunsdkcore.acs_exception.exceptions import ClientException\n", + "from aliyunsdkcore.acs_exception.exceptions import ServerException\n", + "# 加载获取 、 新增、 更新、 删除接口\n", + "from aliyunsdkalidns.request.v20150109 import DescribeDomainsRequest,DescribeSubDomainRecordsRequest, AddDomainRecordRequest, UpdateDomainRecordRequest, DeleteDomainRecordRequest\n", + "# 加载内置模块\n", + "import json,urllib\n", + "\n", + "# AccessKey 和 Secret 建议使用 RAM 子账户的 KEY 和 SECRET 增加安全性\n", + "ID = 'LTAI5tJwE7yDqpAgsDSLN5AB'\n", + "SECRET = 'we5xvM3e27dOKNiGlyhYMFPCClFlfh'\n", + "# 地区节点 可选地区取决于你的阿里云帐号等级，普通用户只有四个，分别是杭州、上海、深圳、河北，具体参考官网API\n", + "regionId = 'cn-hangzhou'\n", + "# 配置认证信息\n", + "client = AcsClient(ID, SECRET, regionId)\n", + "request = DescribeDomainsRequest.DescribeDomainsRequest()\n", + "request.set_accept_format('json')\n", + "response = client.do_action_with_exception(request)\n", + "response = str(response, encoding='utf-8')\n", + "print('域名列表',response)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b5d8bb40-db6f-4691-ba43-77128f853d36", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}