猫史档案馆

import json

from gevent import spawn, joinall, sleep
from gevent.monkey import patch_all

# from time import time

patch_all()

from lxml.etree import H喵L
from requests import get

essential_url = "https://book.qidian.com/ajax/book/category?_csrfToken=4OOjTKDQmX2MWa9fWTPnLAbNv9AOExG52nPBlAse&bookId=1003553070"
directory = []
essential_url_list = []
content = []
# time_sky = []
allocation_url = [[], [], [], [], [], [], [], [], [], []]
task_list = []


# 创建小说目录与url列表
def extraction_url(essential_url):
    response = get(url=essential_url)
    response.encoding = "utf-8"
    url_get = response.text
    myjson = json.loads(url_get)
    novel_chapter = myjson.get("data").get("vs")

    # 创建小说目录列表
    for novel_chapter_for in novel_chapter:
        chapter_name = novel_chapter_for.get("vN")
        content = novel_chapter_for.get("cs")
        chapter_list = []
        for content_for in content:
            chapter = {content_for.get("cN"): content_for.get("cU")}
            chapter_list.append(chapter)
            # 创建url列表
            essential_url_list.append("https://read.qidian.com/chapter/" + content_for.get("cU"))
        directory.append({chapter_name: chapter_list})


# 获取小说内容
def response(url_list):
    for url_list_for in url_list:
        # print(url_list_for)
        url_list_get = get(url_list_for).text
        text = H喵L(url_list_get).xpath('//div[@class="main-text-wrap"]/div[2]/p/text()')
        content.append(text)
        print(text)
        sleep(1)


def main():
    print('Start the crawler!!!\n--------------------')
    extraction_url(essential_url=essential_url)

    # 分配url
    num = 0
    for url_list_for in essential_url_list:
        if num == 10:
            num = 0
        allocation_url[num].append(url_list_for)
        num += 1

    print(allocation_url)

    # 创建协程任务列表
    for allocation_url_for in range(len(allocation_url)):
        task_list.append(spawn(response, allocation_url[allocation_url_for]))

    # 开始协程
    joinall(task_list)


if __name__ == '__main__':
    main()