这一次我们讲讲paddlepadle这个百度开源的机器学习框架，一个影象分类任务从训练到测试出结果的全流程。

将涉及到paddlepaddle和visualdl，git如下：https://github.com/PaddlePaddle

相关的程式码、资料都在我们 Git 上，希望大家 Follow 一下这个 Git 专案，后面会持续更新不同框架下的任务。

https://github.com/longpeng2008/yousan.ai

1 paddlepaddle是什么

sudo pip install paddlepaddle

安装就是一条命令，话不多说上程式码。

2 paddlepaddle训练

2.1 资料定义

定义一个影象分类任务的dataset如下：

from multiprocessing import cpu_count

import paddle.v2 as paddle

class Dataset:

def __init__(self,cropsize,resizesize):

self.cropsize = cropsize

self.resizesize = resizesize

def train_mapper(self,sample):

img, label = sample

img = paddle.image.load_image(img)

img = paddle.image.simple_transform(img, self.resizesize, self.cropsize, True)

#print "train_mapper",img.shape,label

return img.flatten().astype(\'float32\'), label

def test_mapper(self,sample):

img, label = sample

img = paddle.image.load_image(img)

img = paddle.image.simple_transform(img, self.resizesize, self.cropsize, False)

#print "test_mapper",img.shape,label

return img.flatten().astype(\'float32\'), label

def train_reader(self,train_list, buffered_size=1024):

def reader():

with open(train_list, \'r\') as f:

lines = [line.strip() for line in f.readlines()]

print "len of train dataset=",len(lines)

for line in lines:

img_path, lab = line.strip().split(\' \')

yield img_path, int(lab)

return paddle.reader.xmap_readers(self.train_mapper, reader,

cpu_count(), buffered_size)

def test_reader(self,test_list, buffered_size=1024):

def reader():

with open(test_list, \'r\') as f:

lines = [line.strip() for line in f.readlines()]

print "len of val dataset=",len(lines)

for line in lines:

img_path, lab = line.strip().split(\' \')

yield img_path, int(lab)

return paddle.reader.xmap_readers(self.test_mapper, reader,

cpu_count(), buffered_size)

从上面程式码可以看出:

(1) 使用了paddle.image.load_image进行图片的读取，

paddle.image.simple_transform进行了简单的影象变换，这里只有影象crop操作，更多的使用可以参考API。

(2) 使用了paddle.reader.xmap_readers进行资料的对映。

2.2 网络定义

# coding=utf-8

import paddle.fluid as fluid

def simplenet(input):

# 定义卷积块

conv1 = fluid.layers.conv2d(input=input, num_filters=12,stride=2,padding=1,filter_size=3,act="relu")

bn1 = fluid.layers.batch_norm(input=conv1)

conv2 = fluid.layers.conv2d(input=bn1, num_filters=12,stride=2,padding=1,filter_size=3,act="relu")

bn2 = fluid.layers.batch_norm(input=conv2)

conv3 = fluid.layers.conv2d(input=bn2, num_filters=12,stride=2,padding=1,filter_size=3,act="relu")

bn3 = fluid.layers.batch_norm(input=conv3)

fc1 = fluid.layers.fc(input=bn3, size=128, act=None)

return fc1,conv1

与之前的caffe，pytorch，tensorflow框架一样，定义了一个3层卷积与2层全连线的网络。为了能够更好的进行视觉化，我们使用了PaddlePaddle Fluid，Fluid的设计也是用来让使用者像Pytorch和Tensorflow Eager Execution一样可以执行动态计算而不需要建立图。

2.3视觉化

paddlepaddle有与之配套使用的视觉化框架，即visualdl。

visualdl是百度资料视觉化实验室释出的深度学习视觉化平台，它的定位与tensorboard很像，视觉化内容包含了向量，引数直方图分布，模型结构，影象等功能，以后我们会详细给大家讲述，这次直接在程式码中展示如何使用。

安装使用pip install --upgrade visualdl，使用下面的命令可以检视官方例子：

vdl_create_scratch_log

visualDL --logdir ./scratch_log --port 8080

http://127.0.0.1:8080

下面是loss和直方图的检视

在咱们专案中，具体使用方法如下

# 首先定义相关变数

# 建立VisualDL，并指定log储存路径

logdir = "./logs"

logwriter = LogWriter(logdir, sync_cycle=10)

# 建立loss的趋势图

with logwriter.mode("train") as writer:

loss_scalar = writer.scalar("loss")

# 建立acc的趋势图

with logwriter.mode("train") as writer:

acc_scalar = writer.scalar("acc")

# 定义输出频率

num_samples = 4

# 建立卷积层和输出影象的图形化展示

with logwriter.mode("train") as writer:

conv_image = writer.image("conv_image", num_samples, 1)

input_image = writer.image("input_image", num_samples, 1)

# 建立视觉化的训练模型结构

with logwriter.mode("train") as writer:

param1_histgram = writer.histogram("param1", 100)

然后在训练过程中进行记录，这是完整的训练程式码，红色部分就是记录结果。

# coding=utf-8

import numpy as np

import os

import paddle.fluid as fluid

import paddle.fluid.framework as framework

import paddle.v2 as paddle

from paddle.fluid.initializer import NormalInitializer

from paddle.fluid.param_attr import ParamAttr

from visualdl import LogWriter

from dataset import Dataset

from net_fluid import simplenet

# 建立VisualDL，并指定当前该专案的VisualDL的路径

logdir = "./logs"

logwriter = LogWriter(logdir, sync_cycle=10)

# 建立loss的趋势图

with logwriter.mode("train") as writer:

loss_scalar = writer.scalar("loss")

# 建立acc的趋势图

with logwriter.mode("train") as writer:

acc_scalar = writer.scalar("acc")

# 定义输出频率

num_samples = 4

# 建立卷积层和输出影象的图形化展示

with logwriter.mode("train") as writer:

conv_image = writer.image("conv_image", num_samples, 1)

input_image = writer.image("input_image", num_samples, 1)

# 建立视觉化的训练模型结构

with logwriter.mode("train") as writer:

param1_histgram = writer.histogram("param1", 100)

def train(use_cuda, learning_rate, num_passes, BATCH_SIZE=128):

class_dim = 2

image_shape = [3, 48, 48]

image = fluid.layers.data(name=\'image\', shape=image_shape, dtype=\'float32\')

label = fluid.layers.data(name=\'label\', shape=[1], dtype=\'int64\')

net, conv1 = simplenet(image)

# 获取全连线输出

predict = fluid.layers.fc(

input=net,

size=class_dim,

act=\'softmax\',

param_attr=ParamAttr(name="param1", initializer=NormalInitializer()))

# 获取损失

cost = fluid.layers.cross_entropy(input=predict, label=label)

avg_cost = fluid.layers.mean(x=cost)

# 计算batch，从而来求平均的准确率

batch_size = fluid.layers.create_tensor(dtype=\'int64\')

print "batchsize=",batch_size

batch_acc = fluid.layers.accuracy(input=predict, label=label, total=batch_size)

# 定义优化方法

optimizer = fluid.optimizer.Momentum(

learning_rate=learning_rate,

momentum=0.9,

regularization=fluid.regularizer.L2Decay(5 * 1e-5))

opts = optimizer.minimize(avg_cost)

# 是否使用GPU

place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()

# 建立侦错程式

exe = fluid.Executor(place)

# 初始化侦错程式

exe.run(fluid.default_startup_program())

# 储存结果

model_save_dir = "./models"

# 获取训练资料

resizesize = 60

cropsize = 48

mydata = Dataset(cropsize=cropsize,resizesize=resizesize)

mydatareader = mydata.train_reader(train_list=\'./all_shuffle_train.txt\')

train_reader = paddle.batch(reader=paddle.reader.shuffle(reader=mydatareader,buf_size=50000),batch_size=128)

# 指定资料和label的对应关系

feeder = fluid.DataFeeder(place=place, feed_list=[image, label])

step = 0

sample_num = 0

start_up_program = framework.default_startup_program()

param1_var = start_up_program.global_block().var("param1")

accuracy = fluid.average.WeightedAverage()

# 开始训练，使用循环的方式来指定训多少个Pass

for pass_id in range(num_passes):

# 从训练资料中按照一个个batch来读取资料

accuracy.reset()

for batch_id, data in enumerate(train_reader()):

loss, conv1_out, param1, acc, weight = exe.run(fluid.default_main_program(),

feed=feeder.feed(data),

fetch_list=[avg_cost, conv1, param1_var, batch_acc,batch_size])

accuracy.add(value=acc, weight=weight)

pass_acc = accuracy.eval()

# 重新启动图形化展示元件

if sample_num == 0:

input_image.start_sampling()

conv_image.start_sampling()

# 获取taken

idx1 = input_image.is_sample_taken()

idx2 = conv_image.is_sample_taken()

# 保证它们的taken是一样的

assert idx1 == idx2

idx = idx1

if idx != -1:

# 载入输入影象的资料资料

image_data = data[0][0]

input_image_data = np.transpose(

image_data.reshape(image_shape), axes=[1, 2, 0])

input_image.set_sample(idx, input_image_data.shape,

input_image_data.flatten())

# 载入卷积资料

conv_image_data = conv1_out[0][0]

conv_image.set_sample(idx, conv_image_data.shape,

conv_image_data.flatten())

# 完成输出一次

sample_num += 1

if sample_num % num_samples == 0:

input_image.finish_sampling()

conv_image.finish_sampling()

sample_num = 0

# 载入趋势图的资料

loss_scalar.add_record(step, loss)

acc_scalar.add_record(step, acc)

# 新增模型结构资料

param1_histgram.add_record(step, param1.flatten())

# 输出训练日志

print("loss:" + str(loss) + " acc:" + str(acc) + " pass_acc:" + str(pass_acc))

step += 1

model_path = os.path.join(model_save_dir,str(pass_id))

if not os.path.exists(model_save_dir):

os.mkdir(model_save_dir)

fluid.io.save_inference_model(model_path,[\'image\'],[predict],exe)

if __name__ == \'__main__\':

# 开始训练

train(use_cuda=False, learning_rate=0.005, num_passes=300)

2.4训练结果

看看acc和loss的曲线，可见已经收敛

3 paddlepaddle测试

# encoding:utf-8

import sys

import numpy as np

import paddle.v2 as paddle

from PIL import Image

import os

import cv2

# coding=utf-8

import numpy as np

import paddle.fluid as fluid

import paddle.fluid.framework as framework

import paddle.v2 as paddle

from paddle.fluid.initializer import NormalInitializer

from paddle.fluid.param_attr import ParamAttr

from visualdl import LogWriter

from net_fluid import simplenet

if __name__ == "__main__":

# 开始预测

type_size = 2

testsize = 48

imagedir = sys.argv[1]

images = os.listdir(imagedir)

# 定义侦错程式

save_dirname = "./models/299"

exe = fluid.Executor(fluid.CPUPlace())

inference_scope = fluid.core.Scope()

with fluid.scope_guard(inference_scope):

# 载入模型

[inference_program,feed_target_names,fetch_targets] = fluid.io.load_inference_model(save_dirname,exe)

predicts = np.zeros((type_size,1))

for image in images:

imagepath = os.path.join(imagedir,image)

img = paddle.image.load_image(imagepath)

img = paddle.image.simple_transform(img,testsize,testsize,False)

img = img[np.newaxis,:]

#print img.shape

results = np.argsort(-exe.run(inference_program,feed={feed_target_names[0]:img},

fetch_list=fetch_targets)[0])

label = results[0][0]

predicts[label] += 1

print predicts

由于所有框架的测试流程都差不多，所以就不对每一部分进行解释了，大家可以自行去看程式码。