%matplotlib inline
from d2l import tensorflow as d2l
import tensorflow as tf
from tensorflow import keras
import numpy as np
def cls_predictor(num_anchors, num_classes):
return keras.layers.Conv2D(num_anchors * (num_classes + 1),
kernel_size=3, padding='same')Single Shot Detection
Single Shot MultiBox Detector (Liu et al., 2016) is the prototype single-stage detector. One forward pass produces class scores and box offsets for every anchor at every scale; NMS keeps the survivors.
The architecture: a CNN trunk, then a pyramid of feature maps at decreasing resolutions. Each level has its own pair of 1×1-style heads — one for class scores, one for box offsets. Predictions from all levels are concatenated.
SSD = base network + several multiscale feature blocks; each block has its own anchor predictor.
Training
Standard SGD loop, two evaluation metrics (class accuracy, box mean abs error). Read them together: class accuracy is dominated by many background anchors, while box error only makes sense on matched positive anchors.
num_epochs, timer = 20, d2l.Timer()
animator = d2l.Animator(xlabel='epoch', xlim=[1, num_epochs],
legend=['class error', 'bbox mae'])
# Anchors depend only on the input *shape*, so for a fixed input size
# they're constant across the epoch. Grab them once here and reuse
# them in every batch's multibox_target call — saves one forward pass
# per step.
sample_features, _ = next(iter(train_iter))
sample_features = tf.cast(sample_features, tf.float32)
anchors_static, _, _ = net(sample_features, training=False)
# `multibox_target` mixes TF ops with NumPy / `.numpy()` calls, which
# can't run inside `@tf.function`. We keep it eager and only put the
# pure-TF gradient step under @tf.function. The wrapper closes over
# `net` so its trainable variables aren't @tf.function arguments
# (which would re-trace each call) — same pattern that worked for
# gan / dcgan.
@tf.function(reduce_retracing=True)
def _grad_step(features, bbox_labels, bbox_masks, cls_labels):
with tf.GradientTape() as tape:
_, cls_preds, bbox_preds = net(features, training=True)
loss = net._compute_ssd_loss(cls_preds, cls_labels, bbox_preds,
bbox_labels, bbox_masks)
grads = tape.gradient(loss, net.trainable_variables)
net.optimizer.apply_gradients(zip(grads, net.trainable_variables))
cls_correct = tf.reduce_sum(
tf.cast(tf.argmax(cls_preds, axis=-1) ==
tf.cast(cls_labels, tf.int64), tf.int64))
cls_total = tf.cast(tf.size(cls_labels), tf.int64)
bbox_abs = tf.reduce_sum(
tf.abs((bbox_preds * bbox_masks) - (bbox_labels * bbox_masks)))
bbox_total = tf.cast(tf.size(bbox_labels), tf.float32)
return loss, cls_correct, cls_total, bbox_abs, bbox_total
for epoch in range(num_epochs):
# Accumulate metrics as TF tensors so the inner loop never forces
# a host sync (the per-batch `int(logs['cls_correct'])` was a
# significant bottleneck before).
cls_correct_sum = tf.zeros((), dtype=tf.int64)
cls_total = tf.zeros((), dtype=tf.int64)
bbox_abs_total = tf.zeros((), dtype=tf.float32)
bbox_total = tf.zeros((), dtype=tf.float32)
for features, target in train_iter:
timer.start()
features = tf.cast(features, tf.float32)
target = tf.cast(target, tf.float32)
bbox_labels, bbox_masks, cls_labels = d2l.multibox_target(
anchors_static, target)
_, cc, ct, ba, bt = _grad_step(features, bbox_labels,
bbox_masks, cls_labels)
cls_correct_sum += cc
cls_total += ct
bbox_abs_total += ba
bbox_total += bt
cls_err = float(1 - cls_correct_sum / cls_total)
bbox_mae = float(bbox_abs_total / bbox_total)
animator.add(epoch + 1, (cls_err, bbox_mae))
print(f'class err {cls_err:.2e}, bbox mae {bbox_mae:.2e}')
print(f'{len(train_iter) * batch_size / timer.stop():.1f} examples/sec')
class err 3.46e-03, bbox mae 3.73e-03
1104.0 examples/secDetect bananas
Visualize all predictions with confidence ≥ 0.9. The useful thing to notice is not the raw tensor length, but whether NMS leaves one tight high-confidence box per banana:
def display(img, output, threshold):
d2l.set_figsize((5, 5))
fig = d2l.plt.imshow(img)
for row in output:
score = float(row[1])
if score < threshold:
continue
h, w = img.shape[:2]
bbox = [row[2:6] * np.array((w, h, w, h), dtype=np.float32)]
d2l.show_bboxes(fig.axes, bbox, '%.2f' % score, 'w')
display(img, output, threshold=0.9)
def smooth_l1(data, scalar):
cond = tf.abs(data) < 1 / (scalar ** 2)
quad = (scalar * data) ** 2 / 2
lin = tf.abs(data) - 0.5 / (scalar ** 2)
return tf.where(cond, quad, lin)
sigmas = [10, 1, 0.5]
lines = ['-', '--', '-.']
x = tf.range(-2.0, 2.0, 0.1)
d2l.set_figsize()
for l, s in zip(lines, sigmas):
y = smooth_l1(x, scalar=s)
d2l.plt.plot(x, y, l, label='sigma=%.1f' % s)
d2l.plt.legend();
def focal_loss(gamma, x):
return -(1 - x) ** gamma * tf.math.log(x)
x = tf.range(0.01, 1.0, 0.01)
for l, gamma in zip(lines, [0, 1, 5]):
y = d2l.plt.plot(x, focal_loss(gamma, x), l, label='gamma=%.1f' % gamma)
d2l.plt.legend();