环境：pycharm 2019
有点偷懒，缺少了一些相应的检测
第一步 获目标网页的信息

1. url = https://www.kaola.com/

	from selenium import webdriver
 	url = 'http://www.kaola.com/'
    #不打开浏览器
    # chrome_options = webdriver.ChromeOptions()
    # chrome_options.add_argument('--headless')
    # driver = webdriver.Chrome(chrome_options=chrome_options)
    driver = webdriver.Chrome() #打开浏览器，需要配置selinume
    driver.implicitly_wait(10)#隐式等待十秒，推荐使用显示等待，自行查找资料
    #大致就是 等待某些页面元素加载出来再执行下一步
    driver.get(url)
    html = driver.page_source #返回网站文本信息
    print(html)

这样就可以获取目标网站页面的全部信息，或者在网页中通过开发者工具直接观察
输入框和搜索按钮，找到网页中的输入框以及搜索按钮所在点，之后可以通过ID或者CSS选择器或者Xpath路经将所需要搜索的物品发送进去以及点击，之后都是用Xpath路径

selinume 有个find_element_by_xpath()函数可以通过xpath路径找到所需要的属性，文本等

driver.find_element_by_xpath('//*[@id="topSearchInput"]').send_keys(name)
#关于xpath的路径 这里就不多说 不了解自己查资料吧 
#send_keys()向搜索框输入需要查找的物品
driver.find_element_by_xpath('//*[@id="topSearchBtn"]').click()
#点击

执行到这里之后页面应该会跳转到你搜索的物品的商品界面，那么之后就要分析商品的页面了，因为有些网站是动态的商品的信息不会一下就显示在网页内容里
需要下拉页面

driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

那么我们可以观察到页面已经将所有商品信息都包含在网页中了，因此不需要下拉网页

第二步，获取商品的信息
可以直接使用 find_element_by_xpath()找到所需要的信心，但是我上面为了查看当前页面的源码所以之后我再转换为HTML,HTML文本进行初始化，成功构造XPath解析对象，同时可以自动修正HMTL文本

Chrom有个很好用的开发着工具，Xpath可以查看你的预览xpath路径所查询到的东西。

 	name = result.xpath('//div[@class="img"]/img/@alt')
    price =result.xpath('//div[@class="desc clearfix"]/p[1]/span[1]/text()')
    title = '¥'
    for i in range(len(price)):
        price[i] = title + price[i]
    comments =result.xpath('//div[@class="desc clearfix"]/p[3]//text()')
    comments = ' '.join(comments)
    comments = re.findall(r'\d+',comments)
    storename = result.xpath('//div[@class="desc clearfix"]/p[4]//text()')
    storename_list = []
    for item in storename:
        if item != '' and item != ' ' and item != '  ' and item != '   ':
            storename_list.append(item)
    date = zip(name,price,comments,storename_list)#
    return date
    #获得商品信息后，会发现各种问题，比如列表里有很多空格，或者多了很多莫名的符号，那么要么就是xpath路径有问题，要么就需要自己筛选结果，比如正则表达式等

换页,找到下一页click()点击，按道理应该等到页码显示高亮再执行爬取的操作，但我找没找高亮的位置

driver.find_element_by_xpath('//*[@class="nextPage"]').click()

第三步，保存结果
这里只列举比较特殊的csv以及Mysql的保存方式
pycharm可以直接连接Mysql
CSV格式有挺多坑，比如中文乱码等问题

 with open('2.csv', 'a', encoding='utf-8-sig', newline='') as f: 
  # 保存csv时编码格式 utf-o-sig不然会乱码
        writer = csv.writer(f)
        list = ['商品:','价格','评论','店铺']
        writer.writerow(list)
        for item in data:
            writer.writerow(item)

数据库操作

 	 conn = pymysql.Connect(
        host='localhost',
        port=3306,
        user='root',
        passwd='helloworld',
        db='python',
        charset='utf8'
    )
    table_name = input('the name of sheme')
    cursor = conn.cursor()
    sql = 'CREATE TABLE %s ( number INT PRIMARY KEY AUTO_INCREMENT,name varchar(100) ,price varchar(10) not null,comment int(10) not null,store varchar(40)not null);' % table_name
    try:
        cursor.execute(sql)
        conn.commit()
    except:
        conn.rollback()
    sql = 'alter table %s AUTO_INCREMENT = 1;'%table_name
    try:
        cursor.execute(sql)
        conn.commit()
    except:
        conn.rollback()

将信息插入数据库

     for item in data:
        sql = 'INSERT INTO {name} values (number,%s,%s,%s,%s);'.format(name = table_name)
        try:
            cursor.execute(sql, item)
            conn.commit()
        except:
            conn.rollback()

结果


git-hub源码地址

#import urllib.parse
from selenium import webdriver
# from selenium.webdriver.common.by import By
# from selenium.webdriver.support import expected_conditions as EC
from lxml import etree
import pymysql
import time
import csv
import re
def crow(page,name,type):
    # url = 'http://www.baidu.com'
    url = 'http://www.kaola.com/'
    #不打开浏览器
    # chrome_options = webdriver.ChromeOptions()
    # chrome_options.add_argument('--headless')
    # driver = webdriver.Chrome(chrome_options=chrome_options)
    driver = webdriver.Chrome()
    driver.implicitly_wait(10)
    driver.get(url)
    driver.find_element_by_xpath('//*[@id="topSearchInput"]').send_keys(name)
    # input = driver.find_element_by_id()
    time.sleep(1)
    driver.find_element_by_xpath('//*[@id="topSearchBtn"]').click()
    for i in range(page):
        print('你正在爬第%d页\n'%(i+1))
        html = driver.page_source
        data = get_product(html)
        time.sleep(1)
        if type == 1:
            downland(i,data,name)
        elif type == 2:
            table_name = input('the name of sheme')
            conn, cursor = connect_mysql(table_name)
            insert_into(data,table_name,conn,cursor)
            cursor.close()
            conn.close()
        else:
            show(data)
        time.sleep(1)
        if i < page and driver.find_element_by_xpath('//*[@class="nextPage"]'):
            driver.find_element_by_xpath('//*[@class="nextPage"]').click()
        else:
            i = page

def show(data):
    for item in data:
        print('商品名:'+item[0])
        print('价格'+item[1])
        print('评论:' + item[2])
        print('店铺' + item[3])
def get_product(html):
    result = etree.HTML(html)
    name = result.xpath('//div[@class="img"]/img/@alt')
    price =result.xpath('//div[@class="desc clearfix"]/p[1]/span[1]/text()')
    title = '¥'
    for i in range(len(price)):
        price[i] = title + price[i]
    comments =result.xpath('//div[@class="desc clearfix"]/p[3]//text()')
    comments = ' '.join(comments)
    comments = re.findall(r'\d+',comments)
    storename = result.xpath('//div[@class="desc clearfix"]/p[4]//text()')
    storename_list = []
    for item in storename:
        if item != '' and item != ' ' and item != '  ' and item != '   ':
            storename_list.append(item)
    date = zip(name,price,comments,storename_list)
    return date
def downland(i,data,name):
    with open('%s.csv'%name, 'a', encoding='utf-8-sig', newline='') as f:  # 保存csv时编码格式 utf-o-sig不然会乱码
        writer = csv.writer(f)
        if i == 0:
            list = ['商品:','价格','评论','店铺']
            writer.writerow(list)
        for item in data:
            writer.writerow(item)
def insert_into(data,table_name,conn,cursor):
    for item in data:
        sql = 'INSERT INTO {name} values (number,%s,%s,%s,%s);'.format(name = table_name)
        try:
            cursor.execute(sql, item)
            conn.commit()
        except:
            conn.rollback()
    # for item in data
def connect_mysql(table_name):
    conn = pymysql.Connect(
        host='localhost',
        port=3306,
        user='root',
        passwd='helloworld',
        db='python',
        charset='utf8'
    )
    cursor = conn.cursor()
    sql = 'CREATE TABLE %s ( number INT PRIMARY KEY AUTO_INCREMENT,' \
          'name varchar(100) ,price varchar(10) not null,' \
          'comment int(10) not null,' \
          'store varchar(40)not null);' % table_name
    try:
        cursor.execute(sql)
        conn.commit()
    except:
        conn.rollback()
    sql = 'alter table %s AUTO_INCREMENT = 1;' % table_name
    try:
        cursor.execute(sql)
        conn.commit()
    except:
        conn.rollback()
    return conn,cursor
if __name__=='__main__':
    name = input('please enter the name of the goods')
    page = int(input('please enter the page'))
    type = int(input('what type do you want to download? 1 csv 2 mysql 3 I just want to look'))
    crow(page, name, type)
    print('thanks for you try')