💡💡💡本文独家改进：RMT：一种强大的视觉Backbone，灵活地将显式空间先验集成到具有线性复杂度的视觉主干中，在多个下游任务(分类/检测/分割)上性能表现出色！
💡💡💡Transformer 在各个领域验证了可行性，在多个数据集下能够实现涨点
改进结构图如下：
收录
YOLOv8原创自研
💡💡💡全网独家首发创新（原创），适合paper ！！！
💡💡💡 2024年计算机视觉顶会创新点适用于Yolov5、Yolov7、Yolov8等各个Yolo系列，专栏文章提供每一步步骤和源码，轻松带你上手魔改网络 ！！！
💡💡💡重点：通过本专栏的阅读，后续你也可以设计魔改网络，在网络不同位置（Backbone、head、detect、loss等）进行魔改，实现创新！！！
1.RMT原理介绍
论文： 2309.11523.pdf (arxiv.org)
摘要： 近年来，视觉转换器（Vit）在计算机视觉界获得了越来越多的关注。然而，Vit的核心成分自注意缺乏明确的空间先验，并且具有二次计算复杂度，从而限制了Vit的适用性。为了缓解这些问题，我们从最近自然语言处理领域的保留网络（RetNet）中得到启发，提出了RMT，一种具有明确空间先验的通用视觉主干。具体来说，我们将RetNet的时间衰减机制扩展到空间域，并提出了一种基于曼哈顿距离的空间衰减矩阵，以引入显式的空间优先于自注意。此外，为了在不破坏空间衰减矩阵的情况下减少全局信息建模的计算负担，提出了一种能够适应显式空间先验的注意力分解形式。基于空间衰减矩阵和注意力分解形式，我们可以灵活地将显式空间先验集成到具有线性复杂度的视觉主干中。广泛的实验表明，RMT在各种视觉任务中表现出卓越的性能。具体来说，在没有额外训练数据的情况下，RMT在ImageNet-1K上分别以27M/4.5GFLOPS和96M/18.2GFLOPS实现了84.8%和86.1%的TOP-1 ACC。对于下游任务，RMT在COCO检测任务上实现了54.5个框AP和47.2个掩码AP，在ADE20K语义分割任务上实现了52.8miou.
图2。不同自我注意机制的比较。在MaSA中，较深的颜色代表较小的空间衰减率，而较浅的颜色代表较大的空间衰减率。随距离变化的空间衰减率为模型提供了丰富的空间先验
2.如何将入YOLOv8
2.1 新建ultralytics/nn/backbone/RMT.py
import torch
import torch.nn as nn
from torch.nn.common_types import _size_2_t
import torch.utils.checkpoint as checkpoint
from timm.models.layers import DropPath, to_2tuple, trunc_normal_
import math
import torch
import torch.nn.functional as F
import torch.nn as nn
from timm.models.layers import DropPath, trunc_normal_
from timm.models.vision_transformer import VisionTransformer
from timm.models.registry import register_model
from timm.models.vision_transformer import _cfg
from typing import Tuple, Union
from functools import partial
__all__ = ['RMT_T', 'RMT_S', 'RMT_B', 'RMT_L']
class DWConv2d(nn.Module):
def __init__(self, dim, kernel_size, stride, padding):
super().__init__()
self.conv = nn.Conv2d(dim, dim, kernel_size, stride, padding, groups=dim)
def forward(self, x: torch.Tensor):
'''
x: (b h w c)
'''
x = x.permute(0, 3, 1, 2) # (b c h w)
x = self.conv(x) # (b c h w)
x = x.permute(0, 2, 3, 1) # (b h w c)
return x
class RelPos2d(nn.Module):
def __init__(self, embed_dim, num_heads, initial_value, heads_range):
'''
recurrent_chunk_size: (clh clw)
num_chunks: (nch ncw)
clh * clw == cl
nch * ncw == nc
default: clh==clw, clh != clw is not implemented
'''
super().__init__()
angle = 1.0 / (10000 ** torch.linspace(0, 1, embed_dim // num_heads // 2))
angle = angle.unsqueeze(-1).repeat(1, 2).flatten()
self.initial_value = initial_value
self.heads_range = heads_range
self.num_heads = num_heads
decay = torch.log(
1 - 2 ** (-initial_value - heads_range * torch.arange(num_heads, dtype=torch.float) / num_heads))
self.register_buffer('angle', angle)
self.register_buffer('decay', decay)
def generate_2d_decay(self, H: int, W: int):
'''
generate 2d decay mask, the result is (HW)*(HW)
'''
index_h = torch.arange(H).to(self.decay)
index_w = torch.arange(W).to(self.decay)
grid = torch.meshgrid([index_h, index_w])
grid = torch.stack(grid, dim=-1).reshape(H * W, 2) # (H*W 2)
mask = grid[:, None, :] - grid[None, :, :] # (H*W H*W 2)
mask = (mask.abs()).sum(dim=-1)
mask = mask * self.decay[:, None, None] # (n H*W H*W)
return mask
def generate_1d_decay(self, l: int):
'''
generate 1d decay mask, the result is l*l
'''
index = torch.arange(l).to(self.decay)
mask = index[:, None] - index[None, :] # (l l)
mask = mask.abs() # (l l)
mask = mask * self.decay[:, None, None] # (n l l)
return mask
def forward(self, slen: Tuple[int], activate_recurrent=False, chunkwise_recurrent=False):
'''
slen: (h, w)
h * w == l
recurrent is not implemented
'''
if activate_recurrent:
retention_rel_pos = self.decay.exp()
elif chunkwise_recurrent:
mask_h = self.generate_1d_decay(slen[0])
mask_w = self.generate_1d_decay(slen[1])
retention_rel_pos = (mask_h, mask_w)
else:
mask = self.generate_2d_decay(slen[0], slen[1]) # (n l l)
retention_rel_pos = mask
return retention_rel_pos
class MaSAd(nn.Module):
def __init__(self, embed_dim, num_heads, value_factor=1):
super().__init__()
self.factor = value_factor
self.embed_dim = embed_dim
self.num_heads = num_heads
self.head_dim = self.embed_dim * self.factor // num_heads
self.key_dim = self.embed_dim // num_heads
self.scaling = self.key_dim ** -0.5
self.q_proj = nn.Linear(embed_dim, embed_dim, bias=True)
self.k_proj = nn.Linear(embed_dim, embed_dim, bias=True)
self.v_proj = nn.Linear(embed_dim, embed_dim * self.factor, bias=True)
self.lepe = DWConv2d(embed_dim, 5, 1, 2)
self.out_proj = nn.Linear(embed_dim * self.factor, embed_dim, bias=True)
self.reset_parameters()
def reset_parameters(self):
nn.init.xavier_normal_(self.q_proj.weight, gain=2 ** -2.5)
nn.init.xavier_normal_(self.k_proj.weight, gain=2 ** -2.5)
nn.init.xavier_normal_(self.v_proj.weight, gain=2 ** -2.5)
nn.init.xavier_normal_(self.out_proj.weight)
nn.init.constant_(self.out_proj.bias, 0.0)
def forward(self, x: torch.Tensor, rel_pos, chunkwise_recurrent=False, incremental_state=None):
'''
x: (b h w c)
mask_h: (n h h)
mask_w: (n w w)
'''
bsz, h, w, _ = x.size()
mask_h, mask_w = rel_pos
q = self.q_proj(x)
k = self.k_proj(x)
v = self.v_proj(x)
lepe = self.lepe(v)
k *= self.scaling
qr = q.view(bsz, h, w, self.num_heads, self.key_dim).permute(0, 3, 1, 2, 4) # (b n h w d1)
kr = k.view(bsz, h, w, self.num_heads, self.key_dim).permute(0, 3, 1, 2, 4) # (b n h w d1)
'''
qr: (b n h w d1)
kr: (b n h w d1)
v: (b h w n*d2)
'''
qr_w = qr.transpose(1, 2) # (b h n w d1)
kr_w = kr.transpose(1, 2) # (b h n w d1)
v = v.reshape(bsz, h, w, self.num_heads, -1).permute(0, 1, 3, 2, 4) # (b h n w d2)
qk_mat_w = qr_w @ kr_w.transpose(-1, -2) # (b h n w w)
qk_mat_w = qk_mat_w + mask_w # (b h n w w)
qk_mat_w = torch.softmax(qk_mat_w, -1) # (b h n w w)
v = torch.matmul(qk_mat_w, v) # (b h n w d2)
qr_h = qr.permute(0, 3, 1, 2, 4) # (b w n h d1)
kr_h = kr.permute(0, 3, 1, 2, 4) # (b w n h d1)
v = v.permute(0, 3, 2, 1, 4) # (b w n h d2)
qk_mat_h = qr_h @ kr_h.transpose(-1, -2) # (b w n h h)
qk_mat_h = qk_mat_h + mask_h # (b w n h h)
qk_mat_h = torch.softmax(qk_mat_h, -1) # (b w n h h)
output = torch.matmul(qk_mat_h, v) # (b w n h d2)
output = output.permute(0, 3, 1, 2, 4).flatten(-2, -1) # (b h w n*d2)
output = output + lepe
output = self.out_proj(output)
return output
class MaSA(nn.Module):
def __init__(self, embed_dim, num_heads, value_factor=1):
super().__init__()
self.factor = value_factor
self.embed_dim = embed_dim
self.num_heads = num_heads
self.head_dim = self.embed_dim * self.factor // num_heads
self.key_dim = self.embed_dim // num_heads
self.scaling = self.key_dim ** -0.5
self.q_proj = nn.Linear(embed_dim, embed_dim, bias=True)
self.k_proj = nn.Linear(embed_dim, embed_dim, bias=True)
self.v_proj = nn.Linear(embed_dim, embed_dim * self.factor, bias=True)
self.lepe = DWConv2d(embed_dim, 5, 1, 2)
self.out_proj = nn.Linear(embed_dim * self.factor, embed_dim, bias=True)
self.reset_parameters()
def reset_parameters(self):
nn.init.xavier_normal_(self.q_proj.weight, gain=2 ** -2.5)
nn.init.xavier_normal_(self.k_proj.weight, gain=2 ** -2.5)
nn.init.xavier_normal_(self.v_proj.weight, gain=2 ** -2.5)
nn.init.xavier_normal_(self.out_proj.weight)
nn.init.constant_(self.out_proj.bias, 0.0)
def forward(self, x: torch.Tensor, rel_pos, chunkwise_recurrent=False, incremental_state=None):
'''
x: (b h w c)
rel_pos: mask: (n l l)
'''
bsz, h, w, _ = x.size()
mask = rel_pos
assert h * w == mask.size(1)
q = self.q_proj(x)
k = self.k_proj(x)
v = self.v_proj(x)
lepe = self.lepe(v)
k *= self.scaling
qr = q.view(bsz, h, w, self.num_heads, -1).permute(0, 3, 1, 2, 4) # (b n h w d1)
kr = k.view(bsz, h, w, self.num_heads, -1).permute(0, 3, 1, 2, 4) # (b n h w d1)
qr = qr.flatten(2, 3) # (b n l d1)
kr = kr.flatten(2, 3) # (b n l d1)
vr = v.reshape(bsz, h, w, self.num_heads, -1).permute(0, 3, 1, 2, 4) # (b n h w d2)
vr = vr.flatten(2, 3) # (b n l d2)
qk_mat = qr @ kr.transpose(-1, -2) # (b n l l)
qk_mat = qk_mat + mask # (b n l l)
qk_mat = torch.softmax(qk_mat, -1) # (b n l l)
output = torch.matmul(qk_mat, vr) # (b n l d2)
output = output.transpose(1, 2).reshape(bsz, h, w, -1) # (b h w n*d2)
output = output + lepe
output = self.out_proj(output)
return output
class FeedForwardNetwork(nn.Module):
def __init__(
self,
embed_dim,
ffn_dim,
activation_fn=F.gelu,
dropout=0.0,
activation_dropout=0.0,
layernorm_eps=1e-6,
subln=False,
subconv=False
):
super().__init__()
self.embed_dim = embed_dim
self.activation_fn = activation_fn
self.activation_dropout_module = torch.nn.Dropout(activation_dropout)
self.dropout_module = torch.nn.Dropout(dropout)
self.fc1 = nn.Linear(self.embed_dim, ffn_dim)
self.fc2 = nn.Linear(ffn_dim, self.embed_dim)
self.ffn_layernorm = nn.LayerNorm(ffn_dim, eps=layernorm_eps) if subln else None
self.dwconv = DWConv2d(ffn_dim, 3, 1, 1) if subconv else None
def reset_parameters(self):
self.fc1.reset_parameters()
self.fc2.reset_parameters()
if self.ffn_layernorm is not None:
self.ffn_layernorm.reset_parameters()
def forward(self, x: torch.Tensor):
'''
x: (b h w c)
'''
x = self.fc1(x)
x = self.activation_fn(x)
x = self.activation_dropout_module(x)
if self.dwconv is not None:
residual = x
x = self.dwconv(x)
x = x + residual
if self.ffn_layernorm is not None:
x = self.ffn_layernorm(x)
x = self.fc2(x)
x = self.dropout_module(x)
return x
class RetBlock(nn.Module):
def __init__(self, retention: str, embed_dim: int, num_heads: int, ffn_dim: int, drop_path=0., layerscale=False,
layer_init_values=1e-5):
super().__init__()
self.layerscale = layerscale
self.embed_dim = embed_dim
self.retention_layer_norm = nn.LayerNorm(self.embed_dim, eps=1e-6)
assert retention in ['chunk', 'whole']
if retention == 'chunk':
self.retention = MaSAd(embed_dim, num_heads)
else:
self.retention = MaSA(embed_dim, num_heads)
self.drop_path = DropPath(drop_path)
self.final_layer_norm = nn.LayerNorm(self.embed_dim, eps=1e-6)
self.ffn = FeedForwardNetwork(embed_dim, ffn_dim)
self.pos = DWConv2d(embed_dim, 3, 1, 1)
if layerscale:
self.gamma_1 = nn.Parameter(layer_init_values * torch.ones(1, 1, 1, embed_dim), requires_grad=True)
self.gamma_2 = nn.Parameter(layer_init_values * torch.ones(1, 1, 1, embed_dim), requires_grad=True)
def forward(
self,
x: torch.Tensor,
incremental_state=None,
chunkwise_recurrent=False,
retention_rel_pos=None
):
x = x + self.pos(x)
if self.layerscale:
x = x + self.drop_path(
self.gamma_1 * self.retention(self.retention_layer_norm(x), retention_rel_pos, chunkwise_recurrent,
incremental_state))
x = x + self.drop_path(self.gamma_2 * self.ffn(self.final_layer_norm(x)))
else:
x = x + self.drop_path(
self.retention(self.retention_layer_norm(x), retention_rel_pos, chunkwise_recurrent, incremental_state))
x = x + self.drop_path(self.ffn(self.final_layer_norm(x)))
return x
class PatchMerging(nn.Module):
r""" Patch Merging Layer.
Args:
input_resolution (tuple[int]): Resolution of input feature.
dim (int): Number of input channels.
norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm
"""
def __init__(self, dim, out_dim, norm_layer=nn.LayerNorm):
super().__init__()
self.dim = dim
self.reduction = nn.Conv2d(dim, out_dim, 3, 2, 1)
self.norm = nn.BatchNorm2d(out_dim)
def forward(self, x):
'''
x: B H W C
'''
x = x.permute(0, 3, 1, 2).contiguous() # (b c h w)
x = self.reduction(x) # (b oc oh ow)
x = self.norm(x)
x = x.permute(0, 2, 3, 1) # (b oh ow oc)
return x
class BasicLayer(nn.Module):
""" A basic Swin Transformer layer for one stage.
Args:
dim (int): Number of input channels.
input_resolution (tuple[int]): Input resolution.
depth (int): Number of blocks.
num_heads (int): Number of attention heads.
window_size (int): Local window size.
mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True
qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set.
drop (float, optional): Dropout rate. Default: 0.0
attn_drop (float, optional): Attention dropout rate. Default: 0.0
drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0
norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm
downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None
use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False.
fused_window_process (bool, optional): If True, use one kernel to fused window shift & window partition for acceleration, similar for the reversed part. Default: False
"""
def __init__(self, embed_dim, out_dim, depth, num_heads,
init_value: float, heads_range: float,
ffn_dim=96., drop_path=0., norm_layer=nn.LayerNorm, chunkwise_recurrent=False,
downsample: PatchMerging = None, use_checkpoint=False,
layerscale=False, layer_init_values=1e-5):
super().__init__()
self.embed_dim = embed_dim
self.depth = depth
self.use_checkpoint = use_checkpoint
self.chunkwise_recurrent = chunkwise_recurrent
if chunkwise_recurrent:
flag = 'chunk'
else:
flag = 'whole'
self.Relpos = RelPos2d(embed_dim, num_heads, init_value, heads_range)
# build blocks
self.blocks = nn.ModuleList([
RetBlock(flag, embed_dim, num_heads, ffn_dim,
drop_path[i] if isinstance(drop_path, list) else drop_path, layerscale, layer_init_values)
for i in range(depth)])
# patch merging layer
if downsample is not None:
self.downsample = downsample(dim=embed_dim, out_dim=out_dim, norm_layer=norm_layer)
else:
self.downsample = None
def forward(self, x):
b, h, w, d = x.size()
rel_pos = self.Relpos((h, w), chunkwise_recurrent=self.chunkwise_recurrent)
for blk in self.blocks:
if self.use_checkpoint:
tmp_blk = partial(blk, incremental_state=None, chunkwise_recurrent=self.chunkwise_recurrent,
retention_rel_pos=rel_pos)
x = checkpoint.checkpoint(tmp_blk, x)
else:
x = blk(x, incremental_state=None, chunkwise_recurrent=self.chunkwise_recurrent,
retention_rel_pos=rel_pos)
if self.downsample is not None:
x = self.downsample(x)
return x
class LayerNorm2d(nn.Module):
def __init__(self, dim):
super().__init__()
self.norm = nn.LayerNorm(dim, eps=1e-6)
def forward(self, x: torch.Tensor):
'''
x: (b c h w)
'''
x = x.permute(0, 2, 3, 1).contiguous() # (b h w c)
x = self.norm(x) # (b h w c)
x = x.permute(0, 3, 1, 2).contiguous()
return x
class PatchEmbed(nn.Module):
r""" Image to Patch Embedding
Args:
img_size (int): Image size. Default: 224.
patch_size (int): Patch token size. Default: 4.
in_chans (int): Number of input image channels. Default: 3.
embed_dim (int): Number of linear projection output channels. Default: 96.
norm_layer (nn.Module, optional): Normalization layer. Default: None
"""
def __init__(self, in_chans=3, embed_dim=96, norm_layer=None):
super().__init__()
self.in_chans = in_chans
self.embed_dim = embed_dim
self.proj = nn.Sequential(
nn.Conv2d(in_chans, embed_dim // 2, 3, 2, 1),
nn.BatchNorm2d(embed_dim // 2),
nn.GELU(),
nn.Conv2d(embed_dim // 2, embed_dim // 2, 3, 1, 1),
nn.BatchNorm2d(embed_dim // 2),
nn.GELU(),
nn.Conv2d(embed_dim // 2, embed_dim, 3, 2, 1),
nn.BatchNorm2d(embed_dim),
nn.GELU(),
nn.Conv2d(embed_dim, embed_dim, 3, 1, 1),
nn.BatchNorm2d(embed_dim)
)
def forward(self, x):
B, C, H, W = x.shape
x = self.proj(x).permute(0, 2, 3, 1) # (b h w c)
return x
class VisRetNet(nn.Module):
def __init__(self, in_chans=3, num_classes=1000,
embed_dims=[96, 192, 384, 768], depths=[2, 2, 6, 2], num_heads=[3, 6, 12, 24],
init_values=[1, 1, 1, 1], heads_ranges=[3, 3, 3, 3], mlp_ratios=[3, 3, 3, 3], drop_path_rate=0.1,
norm_layer=nn.LayerNorm,
patch_norm=True, use_checkpoints=[False, False, False, False],
chunkwise_recurrents=[True, True, False, False],
layerscales=[False, False, False, False], layer_init_values=1e-6):
super().__init__()
self.num_classes = num_classes
self.num_layers = len(depths)
self.embed_dim = embed_dims[0]
self.patch_norm = patch_norm
self.num_features = embed_dims[-1]
self.mlp_ratios = mlp_ratios
# split image into non-overlapping patches
self.patch_embed = PatchEmbed(in_chans=in_chans, embed_dim=embed_dims[0],
norm_layer=norm_layer if self.patch_norm else None)
# stochastic depth
dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))] # stochastic depth decay rule
# build layers
self.layers = nn.ModuleList()
for i_layer in range(self.num_layers):
layer = BasicLayer(
embed_dim=embed_dims[i_layer],
out_dim=embed_dims[i_layer + 1] if (i_layer < self.num_layers - 1) else None,
depth=depths[i_layer],
num_heads=num_heads[i_layer],
init_value=init_values[i_layer],
heads_range=heads_ranges[i_layer],
ffn_dim=int(mlp_ratios[i_layer] * embed_dims[i_layer]),
drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])],
norm_layer=norm_layer,
chunkwise_recurrent=chunkwise_recurrents[i_layer],
downsample=PatchMerging if (i_layer < self.num_layers - 1) else None,
use_checkpoint=use_checkpoints[i_layer],
layerscale=layerscales[i_layer],
layer_init_values=layer_init_values
)
self.layers.append(layer)
self.channel = [i.size(1) for i in self.forward(torch.randn(1, 3, 640, 640))]
self.apply(self._init_weights)
def _init_weights(self, m):
if isinstance(m, nn.Linear):
trunc_normal_(m.weight, std=.02)
if isinstance(m, nn.Linear) and m.bias is not None:
nn.init.constant_(m.bias, 0)
elif isinstance(m, nn.LayerNorm):
try:
nn.init.constant_(m.bias, 0)
nn.init.constant_(m.weight, 1.0)
except:
pass
@torch.jit.ignore
def no_weight_decay(self):
return {'absolute_pos_embed'}
@torch.jit.ignore
def no_weight_decay_keywords(self):
return {'relative_position_bias_table'}
def forward(self, x):
input_size = x.size(2)
scale = [4, 8, 16, 32]
features = [None, None, None, None]
x = self.patch_embed(x)
if input_size // x.size(2) in scale:
features[scale.index(input_size // x.size(2))] = x.permute(0, 3, 1, 2)
for layer in self.layers:
x = layer(x)
if input_size // x.size(2) in scale:
features[scale.index(input_size // x.size(2))] = x.permute(0, 3, 1, 2)
return features
def RMT_T():
model = VisRetNet(
embed_dims=[64, 128, 256, 512],
depths=[2, 2, 8, 2],
num_heads=[4, 4, 8, 16],
init_values=[2, 2, 2, 2],
heads_ranges=[4, 4, 6, 6],
mlp_ratios=[3, 3, 3, 3],
drop_path_rate=0.1,
chunkwise_recurrents=[True, True, False, False],
layerscales=[False, False, False, False]
)
model.default_cfg = _cfg()
return model
def RMT_S():
model = VisRetNet(
embed_dims=[64, 128, 256, 512],
depths=[3, 4, 18, 4],
num_heads=[4, 4, 8, 16],
init_values=[2, 2, 2, 2],
heads_ranges=[4, 4, 6, 6],
mlp_ratios=[4, 4, 3, 3],
drop_path_rate=0.15,
chunkwise_recurrents=[True, True, True, False],
layerscales=[False, False, False, False]
)
model.default_cfg = _cfg()
return model
def RMT_B():
model = VisRetNet(
embed_dims=[80, 160, 320, 512],
depths=[4, 8, 25, 8],
num_heads=[5, 5, 10, 16],
init_values=[2, 2, 2, 2],
heads_ranges=[5, 5, 6, 6],
mlp_ratios=[4, 4, 3, 3],
drop_path_rate=0.4,
chunkwise_recurrents=[True, True, True, False],
layerscales=[False, False, True, True],
layer_init_values=1e-6
)
model.default_cfg = _cfg()
return model
def RMT_L():
model = VisRetNet(
embed_dims=[112, 224, 448, 640],
depths=[4, 8, 25, 8],
num_heads=[7, 7, 14, 20],
init_values=[2, 2, 2, 2],
heads_ranges=[6, 6, 6, 6],
mlp_ratios=[4, 4, 3, 3],
drop_path_rate=0.5,
chunkwise_recurrents=[True, True, True, False],
layerscales=[False, False, True, True],
layer_init_values=1e-6
)
model.default_cfg = _cfg()
return model
if __name__ == '__main__':
model = RMT_T()
inputs = torch.randn((1, 3, 640, 640))
res = model(inputs)
for i in res:
print(i.size())2.2 注册ultralytics/nn/tasks.py
1)
from ultralytics.nn.backbone.RMT import *2)修改def parse_model(d, ch, verbose=True): # model_dict, input_channels(3)
建议直接替换
def parse_model(d, ch, verbose=True): # model_dict, input_channels(3)
"""Parse a YOLO model.yaml dictionary into a PyTorch model."""
import ast
# Args
max_channels = float("inf")
nc, act, scales = (d.get(x) for x in ("nc", "activation", "scales"))
depth, width, kpt_shape = (d.get(x, 1.0) for x in ("depth_multiple", "width_multiple", "kpt_shape"))
if scales:
scale = d.get("scale")
if not scale:
scale = tuple(scales.keys())[0]
LOGGER.warning(f"WARNING ⚠️ no model scale passed. Assuming scale='{scale}'.")
depth, width, max_channels = scales[scale]
if act:
Conv.default_act = eval(act) # redefine default activation, i.e. Conv.default_act = nn.SiLU()
if verbose:
LOGGER.info(f"{colorstr('activation:')} {act}") # print
if verbose:
LOGGER.info(f"\n{'':>3}{'from':>20}{'n':>3}{'params':>10} {'module':<45}{'arguments':<30}")
ch = [ch]
layers, save, c2 = [], [], ch[-1] # layers, savelist, ch out
is_backbone = False
for i, (f, n, m, args) in enumerate(d["backbone"] + d["head"]): # from, number, module, args
try:
if m == 'node_mode':
m = d[m]
if len(args) > 0:
if args[0] == 'head_channel':
args[0] = int(d[args[0]])
t = m
m = getattr(torch.nn, m[3:]) if 'nn.' in m else globals()[m] # get module
except:
pass
for j, a in enumerate(args):
if isinstance(a, str):
with contextlib.suppress(ValueError):
try:
args[j] = locals()[a] if a in locals() else ast.literal_eval(a)
except:
args[j] = a
n = n_ = max(round(n * depth), 1) if n > 1 else n # depth gain
if m in (
Classify,
Conv,
ConvTranspose,
GhostConv,
Bottleneck,
GhostBottleneck,
SPP,
SPPF,
DWConv,
Focus,
BottleneckCSP,
C1,
C2,
C2f,
C2fAttn,
C3,
C3TR,
C3Ghost,
nn.ConvTranspose2d,
DWConvTranspose2d,
C3x,
RepC3
):
c1, c2 = ch[f], args[0]
if c2 != nc: # if c2 not equal to number of classes (i.e. for Classify() output)
c2 = make_divisible(min(c2, max_channels) * width, 8)
if m is C2fAttn:
args[1] = make_divisible(min(args[1], max_channels // 2) * width, 8) # embed channels
args[2] = int(
max(round(min(args[2], max_channels // 2 // 32)) * width, 1) if args[2] > 1 else args[2]
) # num heads
args = [c1, c2, *args[1:]]
if m in (BottleneckCSP, C1, C2, C2f, C2fAttn, C3, C3TR, C3Ghost, C3x, RepC3):
args.insert(2, n) # number of repeats
n = 1
elif m is AIFI:
args = [ch[f], *args]
elif m is DySample:
c2 = ch[f]
args = [c2, *args]
elif m in (HGStem, HGBlock):
c1, cm, c2 = ch[f], args[0], args[1]
args = [c1, cm, c2, *args[2:]]
if m is HGBlock:
args.insert(4, n) # number of repeats
n = 1
elif m is ResNetLayer:
c2 = args[1] if args[3] else args[1] * 4
elif m is nn.BatchNorm2d:
args = [ch[f]]
elif m is Concat:
c2 = sum(ch[x] for x in f)
elif m in (Detect, WorldDetect, Segment, Pose, OBB, ImagePoolingAttn):
args.append([ch[x] for x in f])
if m is Segment:
args[2] = make_divisible(min(args[2], max_channels) * width, 8)
elif m is RTDETRDecoder: # special case, channels arg must be passed in index 1
args.insert(1, [ch[x] for x in f])
##### backbone
elif isinstance(m, str):
t = m
if len(args) == 2:
m = timm.create_model(m, pretrained=args[0], pretrained_cfg_overlay={'file':args[1]}, features_only=True)
elif len(args) == 1:
m = timm.create_model(m, pretrained=args[0], features_only=True)
c2 = m.feature_info.channels()
elif m in {
RMT_T, RMT_S, RMT_B, RMT_L
}:
m = m(*args)
c2 = m.channel
##### backbone
else:
c2 = ch[f]
if isinstance(c2, list):
is_backbone = True
m_ = m
m_.backbone = True
else:
m_ = nn.Sequential(*(m(*args) for _ in range(n))) if n > 1 else m(*args) # module
t = str(m)[8:-2].replace('__main__.', '') # module type
m.np = sum(x.numel() for x in m_.parameters()) # number params
m_.i, m_.f, m_.type = i + 4 if is_backbone else i, f, t # attach index, 'from' index, type
if verbose:
LOGGER.info(f'{i:>3}{str(f):>20}{n_:>3}{m.np:10.0f} {t:<45}{str(args):<30}') # print
save.extend(x % (i + 4 if is_backbone else i) for x in ([f] if isinstance(f, int) else f) if x != -1) # append to savelist
layers.append(m_)
if i == 0:
ch = []
if isinstance(c2, list):
ch.extend(c2)
for _ in range(5 - len(ch)):
ch.insert(0, 0)
else:
ch.append(c2)
return nn.Sequential(*layers), sorted(save)3）修改def _predict_once(self, x, profile=False, visualize=False, embed=None):
建议直接替换
def _predict_once(self, x, profile=False, visualize=False, embed=None):
"""
Perform a forward pass through the network.
Args:
x (torch.Tensor): The input tensor to the model.
profile (bool): Print the computation time of each layer if True, defaults to False.
visualize (bool): Save the feature maps of the model if True, defaults to False.
embed (list, optional): A list of feature vectors/embeddings to return.
Returns:
(torch.Tensor): The last output of the model.
"""
y, dt, embeddings = [], [], [] # outputs
for m in self.model:
if m.f != -1: # if not from previous layer
x = y[m.f] if isinstance(m.f, int) else [x if j == -1 else y[j] for j in m.f] # from earlier layers
if profile:
self._profile_one_layer(m, x, dt)
if hasattr(m, 'backbone'):
x = m(x)
for _ in range(5 - len(x)):
x.insert(0, None)
for i_idx, i in enumerate(x):
if i_idx in self.save:
y.append(i)
else:
y.append(None)
# for i in x:
# if i is not None:
# print(i.size())
x = x[-1]
else:
x = m(x) # run
y.append(x if m.i in self.save else None) # save output
if visualize:
feature_visualization(x, m.type, m.i, save_dir=visualize)
if embed and m.i in embed:
embeddings.append(nn.functional.adaptive_avg_pool2d(x, (1, 1)).squeeze(-1).squeeze(-1)) # flatten
if m.i == max(embed):
return torch.unbind(torch.cat(embeddings, 1), dim=0)
return x2.3 yolov8-RMT.yaml
# Ultralytics YOLO 🚀, AGPL-3.0 license
# YOLOv8 object detection model with P3-P5 outputs. For Usage examples see https://docs.ultralytics.com/tasks/detect
# Parameters
nc: 80 # number of classes
scales: # model compound scaling constants, i.e. 'model=yolov8n.yaml' will call yolov8.yaml with scale 'n'
# [depth, width, max_channels]
n: [0.33, 0.25, 1024] # YOLOv8n summary: 225 layers, 3157200 parameters, 3157184 gradients, 8.9 GFLOPs
s: [0.33, 0.50, 1024] # YOLOv8s summary: 225 layers, 11166560 parameters, 11166544 gradients, 28.8 GFLOPs
m: [0.67, 0.75, 768] # YOLOv8m summary: 295 layers, 25902640 parameters, 25902624 gradients, 79.3 GFLOPs
l: [1.00, 1.00, 512] # YOLOv8l summary: 365 layers, 43691520 parameters, 43691504 gradients, 165.7 GFLOPs
x: [1.00, 1.25, 512] # YOLOv8x summary: 365 layers, 68229648 parameters, 68229632 gradients, 258.5 GFLOPs
# 0-P1/2
# 1-P2/4
# 2-P3/8
# 3-P4/16
# 4-P5/32
# YOLOv8.0n backbone
backbone:
# [from, repeats, module, args]
- [-1, 1, RMT_T , []] # 4
- [-1, 1, SPPF, [1024, 5]] # 5
# YOLOv8.0n head
head:
- [-1, 1, nn.Upsample, [None, 2, 'nearest']] # 6
- [[-1, 3], 1, Concat, [1]] # 7 cat backbone P4
- [-1, 3, C2f, [512]] # 8
- [-1, 1, nn.Upsample, [None, 2, 'nearest']] # 9
- [[-1, 2], 1, Concat, [1]] # 10 cat backbone P3
- [-1, 3, C2f, [256]] # 11 (P3/8-small)
- [-1, 1, Conv, [256, 3, 2]] # 12
- [[-1, 8], 1, Concat, [1]] # 13 cat head P4
- [-1, 3, C2f, [512]] # 14 (P4/16-medium)
- [-1, 1, Conv, [512, 3, 2]] # 15
- [[-1, 5], 1, Concat, [1]] # 16 cat head P5
- [-1, 3, C2f, [1024]] # 17 (P5/32-large)
- [[11, 14, 17], 1, Detect, [nc]] # Detect(P3, P4, P5)