如果你的PyTorch优化器效果欠佳，试试这四种深度学习中的高级优化技术吧

from functools import partial
 from collections import defaultdict
 import torch
 import torch.nn as nn
 import torch.optim as optim
 import torch.nn.functional as F
 import numpy as np
 import scipy.optimize as opt
 import matplotlib.pyplot as plt
 
 # 设置随机种子以确保结果可复现
 torch.manual_seed(42)
 np.random.seed(42)  
 
 # 辅助函数：在PyTorch模型和NumPy向量之间转换权重
 def set_model_weights_from_vector(model, numpy_vector):
     weight_vector = torch.tensor(numpy_vector, dtype=torch.float64)
     model[0].weight.data = weight_vector[0:4].reshape(2, 2)
     model[2].weight.data = weight_vector[4:8].reshape(2, 2)
     model[2].bias.data = weight_vector[8:10]
     return model
 
 def get_vector_from_model_weights(model):
     return torch.cat([
         model[0].weight.data.view(-1),
         model[2].weight.data.view(-1),
         model[2].bias.data]
    ).detach().numpy()
 
 # 用于跟踪和更新损失的函数
 def update_tracker(loss_tracker, optimizer_name, loss_val):
     loss_tracker[optimizer_name].append(loss_val)
     if len(loss_tracker[optimizer_name]) > 1:
         min_loss = min(loss_tracker[optimizer_name][-2], loss_val)
         loss_tracker[optimizer_name][-1] = min_loss
     return loss_tracker
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.

def objective(x, model, input, target, loss_tracker, optimizer_name):
     model = set_model_weights_from_vector(model, x)
     loss_val = F.mse_loss(model(input), target).item()
     loss_tracker = update_tracker(loss_tracker, optimizer_name, loss_val)  
     return loss_val
 
 def pytorch_optimize(x, model, input, target, maxiter, loss_tracker, optimizer_name="Adam"):
     set_model_weights_from_vector(model, x)
     optimizer = optim.Adam(model.parameters(), lr=1.)
 
     # 训练循环
     for iteration in range(maxiter):  
         loss = F.mse_loss(model(input), target)
         optimizer.zero_grad()  
         loss.backward()        
         optimizer.step()
         loss_tracker = update_tracker(loss_tracker, optimizer_name, loss.item())  
     final_x = get_vector_from_model_weights(model)
     return final_x, loss.item()
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.

model = nn.Sequential(nn.Linear(2, 2, bias=False), nn.ReLU(), nn.Linear(2, 2, bias=True)).double()
 input_tensor = torch.randn(32, 2).double()  # 随机输入张量
 input_tensor[:, 1] *= 1e3  # 增加一个变量的敏感度
 target = input_tensor.clone() # 目标是输入本身（恒等函数）
 num_params = 10
 maxiter = 100
 x0 = 0.1 * np.random.randn(num_params)
 loss_tracker = defaultdict(list)
1.
2.
3.
4.
5.
6.
7.
8.

optimizer_name = "PyTorch Adam"
 result = pytorch_optimize(x0, model, input_tensor, target, maxiter, loss_tracker, optimizer_name)
 print(f'Adam优化器最终损失: {result[1]}')
1.
2.
3.

Adam优化器最终损失: 91.85612831226527
1.

optimizer_name = "slsqp"
 args = (model, input_tensor, target, loss_tracker, optimizer_name)
 result = opt.minimize(objective, x0, method=optimizer_name, args=args, options={"maxiter": maxiter, "disp": False, "eps": 0.001})
 print(f"SLSQP优化器最终损失: {result.fun}")
1.
2.
3.
4.

SLSQP优化器最终损失: 3.097042282788268
1.

from pyswarm import pso
 
 lb = -np.ones(num_params)
 ub = np.ones(num_params)
 optimizer_name = 'pso'
 args = (model, input_tensor, target, loss_tracker, optimizer_name)
 result_pso = pso(objective, lb, ub, maxiter=maxiter, args=args)
 print(f"PSO优化器最终损失: {result_pso[1]}")
1.
2.
3.
4.
5.
6.
7.
8.

PSO优化器最终损失: 1.0195048385714032
1.

from cma import CMAEvolutionStrategy
 
 es = CMAEvolutionStrategy(x0, 0.5, {"maxiter": maxiter, "seed": 42})
 optimizer_name = 'cma'
 args = (model, input_tensor, target, loss_tracker, optimizer_name)
 while not es.stop():
     solutions = es.ask()
     object_vals = [objective(x, *args) for x in solutions]
     es.tell(solutions, object_vals)
 print(f"CMA-ES优化器最终损失: {es.result[1]}")
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.

(5_w,10)-aCMA-ES (mu_w=3.2,w_1=45%) in dimension 10 (seed=42, Thu Oct 12 22:03:53 2024)  
 CMA-ES优化器最终损失: 4.084718909553896
1.
2.

from scipy.optimize import dual_annealing
 bounds = [(-1, 1)] * num_params
 optimizer_name = 'simulated_annealing'
 args = (model, input_tensor, target, loss_tracker, optimizer_name)
 result = dual_annealing(objective, bounds, maxiter=maxiter, args=args, initial_temp=1.)
 print(f"SA优化器最终损失: {result.fun}")
1.
2.
3.
4.
5.
6.

SA优化器最终损失: 0.7834294257939689
1.

plt.figure(figsize=(10, 6))
 line_styles = ['-', '--', '-.', ':']
 for i, (optimizer_name, losses) in enumerate(loss_tracker.items()):
     plt.plot(np.linspace(0, maxiter, len(losses)), losses,
              label=optimizer_name,
              linestyle=line_styles[i % len(line_styles)],
              linewidth=5,
    )
 plt.xlabel("Iteration", fontsize=20)
 plt.ylabel("Loss", fontsize=20)
 plt.ylim(1e-1, 1e7)
 plt.yscale('log')
 plt.title("Loss For Different Optimizers", fontsize=20)
 plt.grid(True, linestyle='--', alpha=0.6)
 plt.legend(loc='upper right', fontsize=20)
 plt.tight_layout()
 plt.savefig('optimizers.png')
 plt.show()
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.