猫史档案馆

如下是paddlepaddle代码：

#数据处理部分之前的代码，加入部分数据处理的库
import paddle
import paddle.fluid as fluid
from paddle.fluid.dygraph.nn import Linear
import numpy as np
import os
import gzip
import json
import random
# 声明数据集文件位置
datafile = './work/mnist.json.gz'
print('loading mnist dataset from {} ......'.format(datafile))
# 加载json数据文件
data = json.load(gzip.open(datafile))
print('mnist dataset load done')
# 读取到的数据区分训练集，验证集，测试集
train_set, val_set, eval_set = data

# 数据集相关参数，图片高度IMG_ROWS, 图片宽度IMG_COLS
IMG_ROWS = 28
IMG_COLS = 28

# 打印数据信息
imgs, labels = train_set[0], train_set[1]
print("训练数据集数量: ", len(imgs))

# 观察验证集数量
imgs, labels = val_set[0], val_set[1]
print("验证数据集数量: ", len(imgs))

# 观察测试集数量
imgs, labels = val= eval_set[0], eval_set[1]
print("测试数据集数量: ", len(imgs))

imgs, labels = train_set[0], train_set[1]
print("训练数据集数量: ", len(imgs))
# 获得数据集长度
imgs_length = len(imgs)
# 定义数据集每个数据的序号，根据序号读取数据
index_list = list(range(imgs_length))
# 读入数据时用到的批次大小
BATCHSIZE = 100

# 随机打乱训练数据的索引序号
random.shuffle(index_list)

# 定义数据生成器，返回批次数据
def data_generator():

    imgs_list = []
    labels_list = []
    for i in index_list:
        # 将数据处理成希望的格式，比如类型为float32，shape为[1, 28, 28]
        img = np.reshape(imgs[i], [1, IMG_ROWS, IMG_COLS]).astype('float32')
        label = np.reshape(labels[i], [1]).astype('float32')
        imgs_list.append(img)
        labels_list.append(label)
        if len(imgs_list) == BATCHSIZE:
            # 获得一个batchsize的数据，并返回
            yield np.array(imgs_list), np.array(labels_list)
            # 清空数据读取列表
            imgs_list = []
            labels_list = []

    # 如果剩余数据的数目小于BATCHSIZE，
    # 则剩余数据一起构成一个大小为len(imgs_list)的mini-batch
    if len(imgs_list) > 0:
        yield np.array(imgs_list), np.array(labels_list)
    return data_generator

# 声明数据读取函数，从训练集中读取数据
train_loader = data_generator
# 以迭代的形式读取数据
for batch_id, data in enumerate(train_loader()):
    image_data, label_data = data
    if batch_id == 0:
        # 打印数据shape和类型
        print("打印第一个batch数据的维度:")
        print("图像维度: {}, 标签维度: {}".format(image_data.shape, label_data.shape))
    break


def load_data(mode='train'):
    datafile = './work/mnist.json.gz'
    print('loading mnist dataset from {} ......'.format(datafile))
    # 加载json数据文件
    data = json.load(gzip.open(datafile))
    print('mnist dataset load done')

    # 读取到的数据区分训练集，验证集，测试集
    train_set, val_set, eval_set = data
    if mode == 'train':
        # 获得训练数据集
        imgs, labels = train_set[0], train_set[1]
    elif mode == 'valid':
        # 获得验证数据集
        imgs, labels = val_set[0], val_set[1]
    elif mode == 'eval':
        # 获得测试数据集
        imgs, labels = eval_set[0], eval_set[1]
    else:
        raise Exception("mode can only be one of ['train', 'valid', 'eval']")
    print("训练数据集数量: ", len(imgs))

    # 校验数据
    imgs_length = len(imgs)

    assert len(imgs) == len(labels), \
        "length of train_imgs({}) should be the same as train_labels({})".format(len(imgs), len(label))

    # 获得数据集长度
    imgs_length = len(imgs)

    # 定义数据集每个数据的序号，根据序号读取数据
    index_list = list(range(imgs_length))
    # 读入数据时用到的批次大小
    BATCHSIZE = 100

    # 定义数据生成器
    def data_generator():
        if mode == 'train':
            # 训练模式下打乱数据
            random.shuffle(index_list)
        imgs_list = []
        labels_list = []
        for i in index_list:
            # 将数据处理成希望的格式，比如类型为float32，shape为[1, 28, 28]
            img = np.reshape(imgs[i], [1, IMG_ROWS, IMG_COLS]).astype('float32')
            label = np.reshape(labels[i], [1]).astype('float32')
            imgs_list.append(img)
            labels_list.append(label)
            if len(imgs_list) == BATCHSIZE:
                # 获得一个batchsize的数据，并返回
                yield np.array(imgs_list), np.array(labels_list)
                # 清空数据读取列表
                imgs_list = []
                labels_list = []

        # 如果剩余数据的数目小于BATCHSIZE，
        # 则剩余数据一起构成一个大小为len(imgs_list)的mini-batch
        if len(imgs_list) > 0:
            yield np.array(imgs_list), np.array(labels_list)

    return data_generator


# 数据处理部分之后的代码，数据读取的部分调用Load_data函数
# 定义网络结构，同上一节所使用的网络结构
class MNIST(fluid.dygraph.Layer):
    def __init__(self):
        super(MNIST, self).__init__()
        self.fc = Linear(input_dim=784, output_dim=1, act=None)

    def forward(self, inputs):
        inputs = fluid.layers.reshape(inputs, (-1, 784))
        outputs = self.fc(inputs)
        return outputs


# 训练配置，并启动训练过程
with fluid.dygraph.guard():
    model = MNIST()
    model.train()
    # 调用加载数据的函数
    train_loader = load_data('train')
    optimizer = fluid.optimizer.SGDOptimizer(learning_rate=0.001, parameter_list=model.parameters())
    EPOCH_NUM = 10
    for epoch_id in range(EPOCH_NUM):
        for batch_id, data in enumerate(train_loader()):
            # 准备数据，变得更加简洁
            image_data, label_data = data
            image = fluid.dygraph.to_variable(image_data)
            label = fluid.dygraph.to_variable(label_data)

            # 前向计算的过程
            predict = model(image)

            # 计算损失，取一个批次样本损失的平均值
            loss = fluid.layers.square_error_cost(predict, label)
            avg_loss = fluid.layers.mean(loss)

            # 每训练了200批次的数据，打印下当前Loss的情况
            if batch_id % 200 == 0:
                print("epoch: {}, batch: {}, loss is: {}".format(epoch_id, batch_id, avg_loss.numpy()))

            # 后向传播，更新参数的过程
            avg_loss.backward()
            optimizer.minimize(avg_loss)
            model.clear_gradients()

    # 保存模型参数
    fluid.save_dygraph(model.state_dict(), 'mnist')

with fluid.dygraph.guard():
    model = MNIST()
    model.train()
    # 调用加载数据的函数
    train_loader = load_data('train')
    # 创建异步数据读取器
    place = fluid.CUDAPlace(0)
    data_loader = fluid.io.DataLoader.from_generator(capacity=5, return_list=True)
    data_loader.set_batch_generator(train_loader, places=place)

    optimizer = fluid.optimizer.SGDOptimizer(learning_rate=0.001, parameter_list=model.parameters())
    EPOCH_NUM = 3
    for epoch_id in range(EPOCH_NUM):
        for batch_id, data in enumerate(data_loader):
            image_data, label_data = data
            image = fluid.dygraph.to_variable(image_data)
            label = fluid.dygraph.to_variable(label_data)

            predict = model(image)

            loss = fluid.layers.square_error_cost(predict, label)
            avg_loss = fluid.layers.mean(loss)

            if batch_id % 200 == 0:
                print("epoch: {}, batch: {}, loss is: {}".format(epoch_id, batch_id, avg_loss.numpy()))

            avg_loss.backward()
            optimizer.minimize(avg_loss)
            model.clear_gradients()

    fluid.save_dygraph(model.state_dict(), 'mnist')

 

returns:

loading mnist dataset from ./work/mnist.json.gz ......
mnist dataset load done
训练数据集数量:  50000
验证数据集数量:  10000
测试数据集数量:  10000
训练数据集数量:  50000
打印第一个batch数据的维度:
图像维度: (100, 1, 28, 28), 标签维度: (100, 1)
loading mnist dataset from ./work/mnist.json.gz ......
mnist dataset load done
训练数据集数量:  50000
epoch: 0, batch: 0, loss is: [25.709703]
epoch: 0, batch: 200, loss is: [3.9081054]
epoch: 0, batch: 400, loss is: [3.703332]
epoch: 1, batch: 0, loss is: [3.361189]
epoch: 1, batch: 200, loss is: [4.0800333]
epoch: 1, batch: 400, loss is: [4.1501217]
epoch: 2, batch: 0, loss is: [2.8981855]
epoch: 2, batch: 200, loss is: [3.2542548]
epoch: 2, batch: 400, loss is: [3.6933615]
epoch: 3, batch: 0, loss is: [3.6170535]
epoch: 3, batch: 200, loss is: [3.592928]
epoch: 3, batch: 400, loss is: [5.46819喵]
epoch: 4, batch: 0, loss is: [3.2573965]
epoch: 4, batch: 200, loss is: [3.1328754]
epoch: 4, batch: 400, loss is: [3.7024407]
epoch: 5, batch: 0, loss is: [3.4103332]
epoch: 5, batch: 200, loss is: [3.1788747]
epoch: 5, batch: 400, loss is: [3.038807]
epoch: 6, batch: 0, loss is: [2.5730155]
epoch: 6, batch: 200, loss is: [2.5897837]
epoch: 6, batch: 400, loss is: [4.8062344]
epoch: 7, batch: 0, loss is: [4.049892]
epoch: 7, batch: 200, loss is: [2.7372766]
epoch: 7, batch: 400, loss is: [3.3898103]
epoch: 8, batch: 0, loss is: [2.7637982]
epoch: 8, batch: 200, loss is: [3.7327032]
epoch: 8, batch: 400, loss is: [3.8119893]
epoch: 9, batch: 0, loss is: [3.6236782]
epoch: 9, batch: 200, loss is: [3.6608539]
epoch: 9, batch: 400, loss is: [3.002493]
loading mnist dataset from ./work/mnist.json.gz ......
mnist dataset load done
训练数据集数量:  50000
epoch: 0, batch: 0, loss is: [54.499332]
epoch: 0, batch: 200, loss is: [3.929251]
epoch: 0, batch: 400, loss is: [3.220104]
epoch: 1, batch: 0, loss is: [4.241174]
epoch: 1, batch: 200, loss is: [3.4087605]
epoch: 1, batch: 400, loss is: [4.2421865]
epoch: 2, batch: 0, loss is: [3.7245395]
epoch: 2, batch: 200, loss is: [3.7845209]
epoch: 2, batch: 400, loss is: [3.404271]