猫史档案馆

module.py:

from PySide2.QtCore import *
from PySide2.QtGui import *
from PySide2.QtWidgets import *
from openpyxl import *
import re
import window
import sys
import threading
import requests
import DecryptLogin
import openpyxl as xl


def list2Dmodel(list, lenth=None):
    if lenth == None:
        l = 3
        for i in list:
            if len(i) > l:
                l = len(i)
        model = QStandardItemModel(len(list), l)

    else:
        model = QStandardItemModel(lenth[0], lenth[1])
    for row in range(len(list)):
        for column in range(len(list[row])):
            item = QStandardItem(list[row][column])
            # item.setEditable(False)
            model.setItem(row, column, item)

    return model


class Main(QMainWindow):
    def __init__(self):
        super(Main, self).__init__()
        self.UI = window.Ui_MainWindow()
        self.UI.setupUi(self)
        self.setWindowTitle("京东爬虫")
        self.inlist = []
        self.n_inlist = []
        self.items = [["name", "price", "url"]]
        self.scorepdfpath = ""
        self.inlistm = QStringListModel()
        self.n_inlistm = QStringListModel()
        self.UI._inlist.setModel(self.inlistm)
        self.UI.n_inlist.setModel(self.n_inlistm)

        self.inlistm.setStringList(self.inlist)
        self.n_inlistm.setStringList(self.n_inlist)
        threading.Thread(target=self.cilckevents()).start()

    def cilckevents(self):
        #   self.UI._in.clicked.connect(
        #      lambda: self.addforinlist())
        # self.UI.n_in.clicked.connect(
        #    lambda: self.addforninlist())
        # self.UI.delforin.clicked.connect(lambda: self.delforinlist())
        # self.UI.delfornin.clicked.connect(lambda: self.delforninlist())
        # self.UI.start.clicked.connect(lambda: threading.Thread(target=self.Crawl, daemon=True).start())
        self.UI._in.clicked.connect(self.addforinlist)
        self.UI.n_in.clicked.connect(self.addforninlist)
        self.UI.delforin.clicked.connect(self.delforinlist)
        self.UI.delfornin.clicked.connect(self.delforninlist)
        self.UI.start.clicked.connect(self.Crawl)
        self.UI.clear.clicked.connect(self.clear)
        self.UI.open.triggered.connect(self.open)
        self.UI.new_2.triggered.connect(self.new)
        self.UI.write.clicked.connect(self.save)
        # while True:
        #  print(self.UI.num.value())

    # 这一段是设置包括和不包括的
    @Slot()
    def addforinlist(self):
        text, ok = (QInputDialog.getText(self, '输入', '输入包含'))
        if ok:
            self.inlist.append(text)
            self.inlistm.setStringList(self.inlist)

    @Slot()
    def addforninlist(self):
        text, ok = (QInputDialog.getText(self, '输入', '输入不包含'))
        if ok:
            self.n_inlist.append(text)
            self.n_inlistm.setStringList(self.n_inlist)

    @Slot()
    def delforinlist(self):
        try:
            self.inlist.pop()
        except:
            pass
        self.inlistm.setStringList(self.inlist)

    @Slot()
    def delforninlist(self):
        try:
            self.n_inlist.pop()
        except:
            pass
        self.n_inlistm.setStringList(self.n_inlist)

    # /设置包括和不包括
    @Slot()
    def clear(self):
        self.items.clear()
        self.UI.printview.setModel(list2Dmodel([]))

    # 清除按钮
    @Slot()
    def Crawl(self):
        self.statusBar().showMessage("正在爬取")
        self.UI.start.setEnabled(False)
        _break = False
        self.q = self.UI.lineEdit.text()
        self.num = self.UI.num.value()
        s = 0
        self.UI.chachbar.setMaximum(self.UI.num.value())
        self.headers = {
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win喵; x喵) AppleWebKit/537.36 (KH喵L, like Gecko) Chrome/94.0.4606.71 Safari/537.36 Edg/94.0.992.38'}  # 设置头,反检测
        self.data = {'keyword': self.q, 'wq': self.q, 'page': ' 1', 's': ' 1', 'click': ' 0'}  # 设置数据字符串
        for i in range(1, 1000000):
            if _break:
                break
            self.url = "http" \
                       "s://s" \
                       "earch" \
                       ".jd.co" \
                       "m/Search?" \
                       "keyword={" \
                       "}&wq=%E6%98" \
                       "%BE%E7%A" \
                       "4%BA%E5%99%" \
                       "A8&pvid=&p" \
                       "age={}&s" \
                       "=1&cli" \
                       "ck=0".format(
                self.q, i)  # 设置url
            # print(_break)
            self.get = requests.get(url=self.url, headers=self.headers,
                                    data=self.data)  # 拉取
            self.a = re.findall(r"<div class=\"gl\-i\-wrap\">([\s\S]*?)</div>([\s]*?)</div>([\s]*?)</li>",
                                self.get.text)
            # print(self.get.text)
            #  print(self.get.text)# 寻找商品标签
            if len(self.a) == 0:
                self.items.append([])
                self.statusBar().showMessage("符合条件商品太少，爬取提前结束")
                print(self.items)
                self.UI.start.setEnabled(True)
                return
            for n in range(len(self.a)):
                i = self.a[n][0]
                if "广告" not in i:  # 屏蔽广告
                    print(type(i), i)
                    url = "https:" + re.search("href=\"([\s\S]*?)\"", i).group(1)
                    # print("\turl:\thttps:" + re.search("href=\"([\s\S]*?)\"", i).group(1), end="")  # 拉url
                    # print(url,i)
                    price = re.search("([0-9]+)\.([0-9]+)",
                                      re.search("<i data-price=([\s\S]*?)>([\s\S]*?)</i>", i).group()).group()
                    # print("\tprice:\t" + re.search("([0-9]+)\.([0-9]+)",
                    #                         re.search("<i data-price=\"([0-9]+)\">([\s\S]*?)</i>",
                    #                                   i).group()).group(), end="")  # 拉取价格
                    # 拉取名称这里很复杂,因为有搜索结果上色,而且也没有什么独特的标签,只能从大标签一个一个找
                    name = re.findall("<a target=\"_blank\"([\s\S]*?)</a>", i)
                    name = re.search("<em>([\s\S]*?)</em>", name[1]).group()
                    name = re.findall(">([\s\S]*?)<.*?", name)
                    # print("\tname:" + "".join(name).replace("\n", ""))
                    # 这段是判断包括和不包括和价格的
                    for i in self.inlist:

                        if i not in "".join(name).replace("\n", ""):
                            print("i=", i, "noappendinin:", "".join(name).replace("\n", ""))
                            break
                    else:
                        for i in self.n_inlist:
                            if i in "".join(name).replace("\n", ""):
                                print("i=", i, "noappendinn:", "".join(name).replace("\n", ""))
                                break
                        else:
                            if (float(price) < self.UI.max.value() and float(
                                    price) > self.UI.min.value()) or self.UI.max.value() == 0:
                                self.items.append(["".join(name).replace("\n", ""), price, url])
                                print("added", ["".join(name).replace("\n", ""), price, url])
                            else:
                                print(f"max={self.UI.max.value()},min={self.UI.min.value()}", "noappendinprice:", price)
                            # print( len(self.items[0]) >= self.s)
                            s += 1
                            self.UI.chachbar.setValue(s)
                            self.UI.printview.setModel(list2Dmodel(self.items))
                    print("s:", s, "num:", self.num)
                    print(self.items)
                if s >= self.num:
                    print(self.num)
                    print("break")
                    _break = True
                    break

        self.statusBar().showMessage("爬取完毕")
        print(self.items)
        self.items.append([])
        self.UI.start.setEnabled(True)
        # print(a.group())
        # <em>Redmi 1A 23.8英寸 IPS技术 三微边设计 低蓝光 HDMI接口 电脑办公<font

    # 打开表格
    @Slot()
    def open(self):
        # print(1)
        self.scorepdfpath, _ = QFileDialog.getOpenFileName(self, "打开excel表格", r"C:\\",
                                                           "excel表格(*.xlsx *.xlxm *.xltx *.xltm)")
        if self.scorepdfpath != "":
            self.wb = xl.load_workbook(self.scorepdfpath)
            self.ws = self.wb.active
            self.ws.column_dimensions["A"].width = 70.0
            self.ws.column_dimensions["B"].width = 20.0
            self.ws.column_dimensions["C"].width = 70.0
            self.ws["a1"] = "商品名称"
            self.ws["b1"] = "价格"
            self.ws["c1"] = "url"

    # 新建表格
    @Slot()
    def new(self):
        # 生成文件对话框对象
        dialog = QFileDialog()
        # 设置文件过滤器，这里是任何文件，包括目录噢
        dialog.setFileMode(QFileDialog.Directory)
        # 设置显示文件的模式，这里是详细模式
        dialog.setViewMode(QFileDialog.Detail)
        if dialog.exec_():
            self.scorepdfpath = dialog.selectedFiles()[0] + r"/Unknown.xlsx"
            if self.scorepdfpath != "":
                self.wb = Workbook()
                self.ws = self.wb.active
                self.ws.column_dimensions["A"].width = 70.0
                self.ws.column_dimensions["B"].width = 20.0
                self.ws.column_dimensions["C"].width = 70.0
                self.ws["a1"] = "商品名称"
                self.ws["b1"] = "价格"
                self.ws["c1"] = "url"

    # 保存按钮
    @Slot()
    def save(self):
        print("dsfbhjsdkg")
        if self.scorepdfpath != "":
            model = self.UI.printview.model()
            row = model.rowCount()
            column = model.columnCount()
            for r in range(row):
                print(row)
                for c in range(column):
                    print(column)
                    print(model.item(r, c).text())
                    self.ws.cell(row=r + 1, column=c + 1).value = model.item(r, c).text()
                    if "https://" in model.item(r, c).text():
                        self.ws.cell(row=r + 1, column=c + 1).hyperlink = model.item(r, c).text()

    # 关闭时
    def closeEvent(self, event):
        if self.scorepdfpath == "":
            #  print("exit1")
            event.accept()
        else:
            try:
                # print(self.scorepdfpath)
                self.wb.save(self.scorepdfpath)
                self.wb.close()
                #  print("exit2")
                event.accept()
            except:
                #     print("exit3")
                self._reply = QMessageBox.question(self, "exit", "文件保存失败，是否退出", QMessageBox.Yes | QMessageBox.No,
                                                   QMessageBox.No)
                if self._reply == QMessageBox.Yes:
                    event.accept()
                else:
                    event.ignore()


if __name__ == "__main__":
    app = QApplication(sys.argv)
    main = Main()
    main.show()
    sys.exit(app.exec_())

window.py(使用QT creator制作,非人工手写):

# -*- coding: utf-8 -*-

################################################################################
## Form generated from reading UI file 'form.ui'
##
## Created by: Qt User Interface Compiler version 5.15.2
##
## WARNING! All changes made in this file will be lost when recompiling UI file!
################################################################################

from PySide2.QtCore import *
from PySide2.QtGui import *
from PySide2.QtWidgets import *


class Ui_MainWindow(object):
    def setupUi(self, MainWindow):
        if not MainWindow.objectName():
            MainWindow.setObjectName(u"MainWindow")
        MainWindow.resize(8喵, 566)
        self.exit = QAction(MainWindow)
        self.exit.setObjectName(u"exit")
        self.open = QAction(MainWindow)
        self.open.setObjectName(u"open")
        self.new_2 = QAction(MainWindow)
        self.new_2.setObjectName(u"new_2")
        self.centralwidget = QWidget(MainWindow)
        self.centralwidget.setObjectName(u"centralwidget")
        self._inlist = QListView(self.centralwidget)
        self._inlist.setObjectName(u"_inlist")
        self._inlist.setGeometry(QRect(40, 280, 101, 161))
        self._in = QPushButton(self.centralwidget)
        self._in.setObjectName(u"_in")
        self._in.setGeometry(QRect(40, 210, 101, 41))
        self.n_inlist = QListView(self.centralwidget)
        self.n_inlist.setObjectName(u"n_inlist")
        self.n_inlist.setGeometry(QRect(180, 280, 101, 161))
        self.n_in = QPushButton(self.centralwidget)
        self.n_in.setObjectName(u"n_in")
        self.n_in.setGeometry(QRect(180, 210, 101, 41))
        self.start = QPushButton(self.centralwidget)
        self.start.setObjectName(u"start")
        self.start.setGeometry(QRect(180, 30, 101, 61))
        self.min = QSpinBox(self.centralwidget)
        self.min.setObjectName(u"min")
        self.min.setGeometry(QRect(30, 160, 51, 22))
        self.min.setCursor(QCursor(Qt.IBeamCursor))
        self.min.setMaximum(10000)
        self.max = QSpinBox(self.centralwidget)
        self.max.setObjectName(u"max")
        self.max.setGeometry(QRect(100, 160, 51, 22))
        self.max.setCursor(QCursor(Qt.IBeamCursor))
        self.max.setMaximum(10000)
        self.label = QLabel(self.centralwidget)
        self.label.setObjectName(u"label")
        self.label.setGeometry(QRect(30, 130, 121, 16))
        self.label_2 = QLabel(self.centralwidget)
        self.label_2.setObjectName(u"label_2")
        self.label_2.setGeometry(QRect(90, 170, 21, 16))
        self.lineEdit = QLineEdit(self.centralwidget)
        self.lineEdit.setObjectName(u"lineEdit")
        self.lineEdit.setGeometry(QRect(40, 30, 101, 61))
        self.chachbar = QProgressBar(self.centralwidget)
        self.chachbar.setObjectName(u"chachbar")
        self.chachbar.setGeometry(QRect(430, 30, 291, 61))
        font = QFont()
        font.setKerning(True)
        self.chachbar.setFont(font)
        self.chachbar.setMaximum(100)
        self.chachbar.setValue(0)
        self.clear = QPushButton(self.centralwidget)
        self.clear.setObjectName(u"clear")
        self.clear.setGeometry(QRect(740, 30, 91, 61))
        self.write = QPushButton(self.centralwidget)
        self.write.setObjectName(u"write")
        self.write.setGeometry(QRect(320, 30, 101, 61))
        self.num = QSpinBox(self.centralwidget)
        self.num.setObjectName(u"num")
        self.num.setEnabled(True)
        self.num.setGeometry(QRect(180, 130, 61, 51))
        self.num.setCursor(QCursor(Qt.IBeamCursor))
        self.num.setMaximum(1000)
        self.label_3 = QLabel(self.centralwidget)
        self.label_3.setObjectName(u"label_3")
        self.label_3.setGeometry(QRect(250, 150, 54, 12))
        self.printview = QTableView(self.centralwidget)
        self.printview.setObjectName(u"printview")
        self.printview.setGeometry(QRect(320, 110, 511, 391))
        self.printview.horizontalHeader().setDefaultSectionSize(200)
        self.delforin = QPushButton(self.centralwidget)
        self.delforin.setObjectName(u"delforin")
        self.delforin.setGeometry(QRect(40, 460, 101, 41))
        self.delfornin = QPushButton(self.centralwidget)
        self.delfornin.setObjectName(u"delfornin")
        self.delfornin.setGeometry(QRect(180, 460, 101, 41))
        MainWindow.setCentralWidget(self.centralwidget)
        self.menubar = QMenuBar(MainWindow)
        self.menubar.setObjectName(u"menubar")
        self.menubar.setGeometry(QRect(0, 0, 8喵, 22))
        self.menu = QMenu(self.menubar)
        self.menu.setObjectName(u"menu")
        self.menu_excel = QMenu(self.menu)
        self.menu_excel.setObjectName(u"menu_excel")
        MainWindow.setMenuBar(self.menubar)
        self.statusbar = QStatusBar(MainWindow)
        self.statusbar.setObjectName(u"statusbar")
        MainWindow.setStatusBar(self.statusbar)

        self.menubar.addAction(self.menu.menuAction())
        self.menu.addAction(self.menu_excel.menuAction())
        self.menu.addAction(self.exit)
        self.menu_excel.addAction(self.open)
        self.menu_excel.addAction(self.new_2)

        self.retranslateUi(MainWindow)

        QMetaObject.connectSlotsByName(MainWindow)
    # setupUi

    def retranslateUi(self, MainWindow):
        MainWindow.setWindowTitle(QCoreApplication.translate("MainWindow", u"MainWindow", None))
        self.exit.setText(QCoreApplication.translate("MainWindow", u"\u9000\u51fa", None))
        self.open.setText(QCoreApplication.translate("MainWindow", u"\u6253\u5f00", None))
        self.new_2.setText(QCoreApplication.translate("MainWindow", u"\u65b0\u5efa", None))
        self._in.setText(QCoreApplication.translate("MainWindow", u"\u6dfb\u52a0\u5305\u62ec", None))
        self.n_in.setText(QCoreApplication.translate("MainWindow", u"\u6dfb\u52a0\u4e0d\u5305\u62ec", None))
        self.start.setText(QCoreApplication.translate("MainWindow", u"\u722c", None))
        self.label.setText(QCoreApplication.translate("MainWindow", u"\u4ef7\u683c\u533a\u95f4(0\u4e3a\u4e0d\u9650\u5236)", None))
        self.label_2.setText(QCoreApplication.translate("MainWindow", u"~", None))
        self.lineEdit.setText(QCoreApplication.translate("MainWindow", u"\u4e3b\u5173\u952e\u5b57", None))
        self.clear.setText(QCoreApplication.translate("MainWindow", u"\u6e05\u7a7a", None))
        self.write.setText(QCoreApplication.translate("MainWindow", u"\u5199\u5165\u8868\u683c", None))
        self.label_3.setText(QCoreApplication.translate("MainWindow", u"\u6570\u91cf", None))
        self.delforin.setText(QCoreApplication.translate("MainWindow", u"\u5220\u96喵\u5c3e\u9879", None))
        self.delfornin.setText(QCoreApplication.translate("MainWindow", u"\u5220\u96喵\u5c3e\u9879", None))
        self.menu.setTitle(QCoreApplication.translate("MainWindow", u"\u6587\u4ef6", None))
        self.menu_excel.setTitle(QCoreApplication.translate("MainWindow", u"\u52a0\u8f7dexcel", None))
    # retranslateUi

------------------------------------------------------------------------------------------------------------------