From 3777c4bd111744bb42a4de03ecf6a8d3332a94d7 Mon Sep 17 00:00:00 2001 From: Xi Xu Date: Wed, 11 Dec 2024 13:29:29 +0800 Subject: [PATCH] Create 13 --- 13/1.py | 136 ++++++++++++++++++++++++++++++++++++++++++++++ 13/2.py | 139 +++++++++++++++++++++++++++++++++++++++++++++++ 13/3.py | 125 ++++++++++++++++++++++++++++++++++++++++++ 13/4.py | 144 ++++++++++++++++++++++++++++++++++++++++++++++++ 13/5.py | 154 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 13/6.py | 166 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 864 insertions(+) create mode 100644 13/2.py create mode 100644 13/3.py create mode 100644 13/4.py create mode 100644 13/5.py create mode 100644 13/6.py diff --git a/13/1.py b/13/1.py index e69de29..df2ddc1 100644 --- a/13/1.py +++ b/13/1.py @@ -0,0 +1,136 @@ +import matplotlib.pyplot as plt +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.utils.data import DataLoader +from torchvision import datasets, transforms + +# Super parameters +batch_size = 64 +learning_rate = 0.01 +momentum = 0.5 +EPOCH = 10 + +# Prepare dataset +transform = transforms.Compose( + [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))] +) + +train_dataset = datasets.MNIST( + root="./data/mnist", train=True, download=True, transform=transform +) +test_dataset = datasets.MNIST( + root="./data/mnist", train=False, download=True, transform=transform +) +train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True) +test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False) + + +class Net(torch.nn.Module): + def __init__(self): + super(Net, self).__init__() + self.fc1 = torch.nn.Linear(28 * 28, 512) + self.fc2 = torch.nn.Linear(512, 128) + self.fc3 = torch.nn.Linear(128, 10) + + def forward(self, x): + x = x.view(-1, 28 * 28) + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = self.fc3(x) + return x + + +# Create two models for comparison +model_ce = Net() +model_mse = Net() + +# Create two different loss functions +criterion_ce = nn.CrossEntropyLoss() +criterion_mse = nn.MSELoss() + +# Create optimizers for both models +optimizer_ce = torch.optim.SGD( + model_ce.parameters(), lr=learning_rate, momentum=momentum +) +optimizer_mse = torch.optim.SGD( + model_mse.parameters(), lr=learning_rate, momentum=momentum +) + + +def train(epoch, model, criterion, optimizer, name=""): + model.train() + running_loss = 0.0 + running_total = 0 + running_correct = 0 + + for batch_idx, (inputs, target) in enumerate(train_loader): + optimizer.zero_grad() + outputs = model(inputs) + + # For MSE, convert targets to one-hot encoding + if isinstance(criterion, nn.MSELoss): + target_one_hot = torch.zeros(target.size(0), 10) + target_one_hot.scatter_(1, target.unsqueeze(1), 1) + loss = criterion(outputs, target_one_hot) + else: + loss = criterion(outputs, target) + + loss.backward() + optimizer.step() + + running_loss += loss.item() + _, predicted = torch.max(outputs.data, dim=1) + running_total += target.shape[0] + running_correct += (predicted == target).sum().item() + + avg_loss = running_loss / len(train_loader) + avg_acc = 100 * running_correct / running_total + print( + f"[{epoch + 1} / {EPOCH}]: {name} Training Loss: {avg_loss:.3f}, Training Accuracy: {avg_acc:.2f} %" + ) + + +def test(epoch, model, name=""): + model.eval() + correct = 0 + total = 0 + with torch.no_grad(): + for data in test_loader: + images, labels = data + outputs = model(images) + _, predicted = torch.max(outputs.data, dim=1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + + acc = 100 * correct / total + print( + f"[{epoch + 1} / {EPOCH}]: {name} Accuracy on test set after epoch {epoch + 1}: {acc:.1f} %" + ) + return acc + + +if __name__ == "__main__": + acc_list_ce = [] + acc_list_mse = [] + + for epoch in range(EPOCH): + train(epoch, model_ce, criterion_ce, optimizer_ce, "CrossEntropy") + train(epoch, model_mse, criterion_mse, optimizer_mse, "MSE") + + acc_ce = test(epoch, model_ce, "CrossEntropy") + acc_mse = test(epoch, model_mse, "MSE") + + acc_list_ce.append(acc_ce) + acc_list_mse.append(acc_mse) + + plt.figure(figsize=(10, 6)) + plt.plot(range(1, EPOCH + 1), acc_list_ce, label="CrossEntropy", marker="o") + plt.plot(range(1, EPOCH + 1), acc_list_mse, label="MSE", marker="s") + plt.xlabel("Epoch") + plt.ylabel("Accuracy On TestSet (%)") + plt.title("Comparison of Loss Functions: CrossEntropy vs MSE") + plt.legend() + plt.grid(True) + plt.show() diff --git a/13/2.py b/13/2.py new file mode 100644 index 0000000..d18d6a3 --- /dev/null +++ b/13/2.py @@ -0,0 +1,139 @@ +import matplotlib.pyplot as plt +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.utils.data import DataLoader +from torchvision import datasets, transforms + +# Super parameters +batch_size_small = 64 +learning_rate = 0.01 +momentum = 0.5 +EPOCH = 10 + +# Prepare dataset +transform = transforms.Compose( + [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))] +) + +train_dataset = datasets.MNIST( + root="./data/mnist", train=True, download=True, transform=transform +) +test_dataset = datasets.MNIST( + root="./data/mnist", train=False, download=True, transform=transform +) + +batch_size_full = len(train_dataset) +train_loader_small = DataLoader( + train_dataset, batch_size=batch_size_small, shuffle=True +) +train_loader_full = DataLoader(train_dataset, batch_size=batch_size_full, shuffle=True) +test_loader = DataLoader(test_dataset, batch_size=batch_size_small, shuffle=False) + + +class Net(torch.nn.Module): + def __init__(self): + super(Net, self).__init__() + self.fc1 = torch.nn.Linear(28 * 28, 512) + self.fc2 = torch.nn.Linear(512, 128) + self.fc3 = torch.nn.Linear(128, 10) + + def forward(self, x): + x = x.view(-1, 28 * 28) + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = self.fc3(x) + return x + + +# Create two models +model_small_batch = Net() +model_full_batch = Net() + +# Create optimizers +optimizer_small = torch.optim.SGD( + model_small_batch.parameters(), lr=learning_rate, momentum=momentum +) +optimizer_full = torch.optim.SGD( + model_full_batch.parameters(), lr=learning_rate, momentum=momentum +) + +criterion = torch.nn.CrossEntropyLoss() + + +def train(epoch, model, optimizer, loader, name=""): + model.train() + running_loss = 0.0 + running_total = 0 + running_correct = 0 + + for batch_idx, (inputs, target) in enumerate(loader): + optimizer.zero_grad() + outputs = model(inputs) + loss = criterion(outputs, target) + + loss.backward() + optimizer.step() + + running_loss += loss.item() + _, predicted = torch.max(outputs.data, dim=1) + running_total += target.shape[0] + running_correct += (predicted == target).sum().item() + + avg_loss = running_loss / len(loader) + avg_acc = 100 * running_correct / running_total + print( + f"[{epoch + 1} / {EPOCH}]: {name} Training Loss: {avg_loss:.3f}, Training Accuracy: {avg_acc:.2f} %" + ) + + +def test(epoch, model, name=""): + model.eval() + correct = 0 + total = 0 + with torch.no_grad(): + for data in test_loader: + images, labels = data + outputs = model(images) + _, predicted = torch.max(outputs.data, dim=1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + + acc = 100 * correct / total + print( + f"[{epoch + 1} / {EPOCH}]: {name} Accuracy on test set after epoch {epoch + 1}: {acc:.1f} %" + ) + return acc + + +if __name__ == "__main__": + acc_list_small = [] + acc_list_full = [] + + for epoch in range(EPOCH): + train( + epoch, model_small_batch, optimizer_small, train_loader_small, "Small Batch" + ) + train(epoch, model_full_batch, optimizer_full, train_loader_full, "Full Batch") + + acc_small = test(epoch, model_small_batch, "Small Batch") + acc_full = test(epoch, model_full_batch, "Full Batch") + + acc_list_small.append(acc_small) + acc_list_full.append(acc_full) + + plt.figure(figsize=(10, 6)) + plt.plot( + range(1, EPOCH + 1), + acc_list_small, + label=f"Batch Size={batch_size_small}", + marker="o", + ) + plt.plot(range(1, EPOCH + 1), acc_list_full, label="Full Batch", marker="s") + plt.xlabel("Epoch") + plt.ylabel("Accuracy On TestSet (%)") + plt.title("Comparison of Batch Sizes") + plt.legend() + plt.grid(True) + plt.show() diff --git a/13/3.py b/13/3.py new file mode 100644 index 0000000..018c680 --- /dev/null +++ b/13/3.py @@ -0,0 +1,125 @@ +import matplotlib.pyplot as plt +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.utils.data import DataLoader +from torchvision import datasets, transforms + +# Super parameters +batch_size = 64 +learning_rate = 0.01 +momentum = 0.5 +EPOCH = 10 + +# Prepare dataset +transform = transforms.Compose( + [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))] +) + +train_dataset = datasets.MNIST( + root="./data/mnist", train=True, download=True, transform=transform +) +test_dataset = datasets.MNIST( + root="./data/mnist", train=False, download=True, transform=transform +) +train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True) +test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False) + + +class Net(torch.nn.Module): + def __init__(self): + super(Net, self).__init__() + self.fc1 = torch.nn.Linear(28 * 28, 512) + self.fc2 = torch.nn.Linear(512, 128) + self.fc3 = torch.nn.Linear(128, 10) + + def forward(self, x): + x = x.view(-1, 28 * 28) + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = self.fc3(x) + return x + + +# Create two models for comparison +model_sgd = Net() +model_adam = Net() + +# Create different optimizers +optimizer_sgd = torch.optim.SGD( + model_sgd.parameters(), lr=learning_rate, momentum=momentum +) +optimizer_adam = torch.optim.Adam(model_adam.parameters(), lr=learning_rate) + +criterion = torch.nn.CrossEntropyLoss() + + +def train(epoch, model, optimizer, name=""): + model.train() + running_loss = 0.0 + running_total = 0 + running_correct = 0 + + for batch_idx, (inputs, target) in enumerate(train_loader): + optimizer.zero_grad() + outputs = model(inputs) + loss = criterion(outputs, target) + + loss.backward() + optimizer.step() + + running_loss += loss.item() + _, predicted = torch.max(outputs.data, dim=1) + running_total += target.shape[0] + running_correct += (predicted == target).sum().item() + + avg_loss = running_loss / len(train_loader) + avg_acc = 100 * running_correct / running_total + print( + f"[{epoch + 1} / {EPOCH}]: {name} Training Loss: {avg_loss:.3f}, Training Accuracy: {avg_acc:.2f} %" + ) + + +def test(epoch, model, name=""): + model.eval() + correct = 0 + total = 0 + with torch.no_grad(): + for data in test_loader: + images, labels = data + outputs = model(images) + _, predicted = torch.max(outputs.data, dim=1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + + acc = 100 * correct / total + print( + f"[{epoch + 1} / {EPOCH}]: {name} Accuracy on test set after epoch {epoch + 1}: {acc:.1f} %" + ) + return acc + + +if __name__ == "__main__": + acc_list_sgd = [] + acc_list_adam = [] + + for epoch in range(EPOCH): + train(epoch, model_sgd, optimizer_sgd, "SGD") + train(epoch, model_adam, optimizer_adam, "Adam") + + acc_sgd = test(epoch, model_sgd, "SGD") + acc_adam = test(epoch, model_adam, "Adam") + + acc_list_sgd.append(acc_sgd) + acc_list_adam.append(acc_adam) + + plt.figure(figsize=(10, 6)) + plt.plot(range(1, EPOCH + 1), acc_list_sgd, label="SGD", marker="o") + plt.plot(range(1, EPOCH + 1), acc_list_adam, label="Adam", marker="s") + plt.xlabel("Epoch") + plt.ylabel("Accuracy On TestSet (%)") + plt.title("Comparison of Optimizers: SGD vs Adam") + plt.legend() + plt.grid(True) + plt.show() diff --git a/13/4.py b/13/4.py new file mode 100644 index 0000000..dad505e --- /dev/null +++ b/13/4.py @@ -0,0 +1,144 @@ +import matplotlib.pyplot as plt +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.utils.data import DataLoader +from torchvision import datasets, transforms + +# Super parameters +batch_size = 64 +learning_rate = 0.01 +momentum = 0.5 +EPOCH = 10 + +# Prepare dataset +transform = transforms.Compose( + [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))] +) + +train_dataset = datasets.MNIST( + root="./data/mnist", train=True, download=True, transform=transform +) +test_dataset = datasets.MNIST( + root="./data/mnist", train=False, download=True, transform=transform +) +train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True) +test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False) + + +class FatShortNet(torch.nn.Module): + def __init__(self): + super(FatShortNet, self).__init__() + self.fc1 = torch.nn.Linear(28 * 28, 4096) + self.fc2 = torch.nn.Linear(4096, 10) + + def forward(self, x): + x = x.view(-1, 28 * 28) + x = F.relu(self.fc1(x)) + x = self.fc2(x) + return x + + +class ThinTallNet(torch.nn.Module): + def __init__(self): + super(ThinTallNet, self).__init__() + self.fc1 = torch.nn.Linear(28 * 28, 128) + self.fc2 = torch.nn.Linear(128, 128) + self.fc3 = torch.nn.Linear(128, 128) + self.fc4 = torch.nn.Linear(128, 128) + self.fc5 = torch.nn.Linear(128, 10) + + def forward(self, x): + x = x.view(-1, 28 * 28) + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = F.relu(self.fc3(x)) + x = F.relu(self.fc4(x)) + x = self.fc5(x) + return x + + +# Create two models +model_fat = FatShortNet() +model_thin = ThinTallNet() + +# Create optimizers +optimizer_fat = torch.optim.SGD( + model_fat.parameters(), lr=learning_rate, momentum=momentum +) +optimizer_thin = torch.optim.SGD( + model_thin.parameters(), lr=learning_rate, momentum=momentum +) + +criterion = torch.nn.CrossEntropyLoss() + + +def train(epoch, model, optimizer, name=""): + model.train() + running_loss = 0.0 + running_total = 0 + running_correct = 0 + + for batch_idx, (inputs, target) in enumerate(train_loader): + optimizer.zero_grad() + outputs = model(inputs) + loss = criterion(outputs, target) + + loss.backward() + optimizer.step() + + running_loss += loss.item() + _, predicted = torch.max(outputs.data, dim=1) + running_total += target.shape[0] + running_correct += (predicted == target).sum().item() + + avg_loss = running_loss / len(train_loader) + avg_acc = 100 * running_correct / running_total + print( + f"[{epoch + 1} / {EPOCH}]: {name} Training Loss: {avg_loss:.3f}, Training Accuracy: {avg_acc:.2f} %" + ) + + +def test(epoch, model, name=""): + model.eval() + correct = 0 + total = 0 + with torch.no_grad(): + for data in test_loader: + images, labels = data + outputs = model(images) + _, predicted = torch.max(outputs.data, dim=1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + + acc = 100 * correct / total + print( + f"[{epoch + 1} / {EPOCH}]: {name} Accuracy on test set after epoch {epoch + 1}: {acc:.1f} %" + ) + return acc + + +if __name__ == "__main__": + acc_list_fat = [] + acc_list_thin = [] + + for epoch in range(EPOCH): + train(epoch, model_fat, optimizer_fat, "Fat+Short") + train(epoch, model_thin, optimizer_thin, "Thin+Tall") + + acc_fat = test(epoch, model_fat, "Fat+Short") + acc_thin = test(epoch, model_thin, "Thin+Tall") + + acc_list_fat.append(acc_fat) + acc_list_thin.append(acc_thin) + + plt.figure(figsize=(10, 6)) + plt.plot(range(1, EPOCH + 1), acc_list_fat, label="Fat+Short", marker="o") + plt.plot(range(1, EPOCH + 1), acc_list_thin, label="Thin+Tall", marker="s") + plt.xlabel("Epoch") + plt.ylabel("Accuracy On TestSet (%)") + plt.title("Comparison of Network Structures") + plt.legend() + plt.grid(True) + plt.show() diff --git a/13/5.py b/13/5.py new file mode 100644 index 0000000..3324a2d --- /dev/null +++ b/13/5.py @@ -0,0 +1,154 @@ +import matplotlib.pyplot as plt +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.utils.data import DataLoader +from torchvision import datasets, transforms + +# Super parameters +batch_size = 64 +learning_rate = 0.01 +momentum = 0.5 +EPOCH = 10 + +# Prepare dataset +transform = transforms.Compose( + [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))] +) + +train_dataset = datasets.MNIST( + root="./data/mnist", train=True, download=True, transform=transform +) +test_dataset = datasets.MNIST( + root="./data/mnist", train=False, download=True, transform=transform +) +train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True) +test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False) + + +class Net3Layer(torch.nn.Module): + def __init__(self): + super(Net3Layer, self).__init__() + self.fc1 = torch.nn.Linear(28 * 28, 512) + self.fc2 = torch.nn.Linear(512, 128) + self.fc3 = torch.nn.Linear(128, 10) + + def forward(self, x): + x = x.view(-1, 28 * 28) + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = self.fc3(x) + return x + + +class Net9Layer(torch.nn.Module): + def __init__(self): + super(Net9Layer, self).__init__() + self.fc1 = torch.nn.Linear(28 * 28, 512) + self.fc2 = torch.nn.Linear(512, 512) + self.fc3 = torch.nn.Linear(512, 256) + self.fc4 = torch.nn.Linear(256, 256) + self.fc5 = torch.nn.Linear(256, 128) + self.fc6 = torch.nn.Linear(128, 128) + self.fc7 = torch.nn.Linear(128, 64) + self.fc8 = torch.nn.Linear(64, 64) + self.fc9 = torch.nn.Linear(64, 10) + + def forward(self, x): + x = x.view(-1, 28 * 28) + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = F.relu(self.fc3(x)) + x = F.relu(self.fc4(x)) + x = F.relu(self.fc5(x)) + x = F.relu(self.fc6(x)) + x = F.relu(self.fc7(x)) + x = F.relu(self.fc8(x)) + x = self.fc9(x) + return x + + +# Create two models +model_3layer = Net3Layer() +model_9layer = Net9Layer() + +# Create optimizers +optimizer_3layer = torch.optim.SGD( + model_3layer.parameters(), lr=learning_rate, momentum=momentum +) +optimizer_9layer = torch.optim.SGD( + model_9layer.parameters(), lr=learning_rate, momentum=momentum +) + +criterion = torch.nn.CrossEntropyLoss() + + +def train(epoch, model, optimizer, name=""): + model.train() + running_loss = 0.0 + running_total = 0 + running_correct = 0 + + for batch_idx, (inputs, target) in enumerate(train_loader): + optimizer.zero_grad() + outputs = model(inputs) + loss = criterion(outputs, target) + + loss.backward() + optimizer.step() + + running_loss += loss.item() + _, predicted = torch.max(outputs.data, dim=1) + running_total += target.shape[0] + running_correct += (predicted == target).sum().item() + + avg_loss = running_loss / len(train_loader) + avg_acc = 100 * running_correct / running_total + print( + f"[{epoch + 1} / {EPOCH}]: {name} Training Loss: {avg_loss:.3f}, Training Accuracy: {avg_acc:.2f} %" + ) + + +def test(epoch, model, name=""): + model.eval() + correct = 0 + total = 0 + with torch.no_grad(): + for data in test_loader: + images, labels = data + outputs = model(images) + _, predicted = torch.max(outputs.data, dim=1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + + acc = 100 * correct / total + print( + f"[{epoch + 1} / {EPOCH}]: {name} Accuracy on test set after epoch {epoch + 1}: {acc:.1f} %" + ) + return acc + + +if __name__ == "__main__": + acc_list_3layer = [] + acc_list_9layer = [] + + for epoch in range(EPOCH): + train(epoch, model_3layer, optimizer_3layer, "3-Layer") + train(epoch, model_9layer, optimizer_9layer, "9-Layer") + + acc_3layer = test(epoch, model_3layer, "3-Layer") + acc_9layer = test(epoch, model_9layer, "9-Layer") + + acc_list_3layer.append(acc_3layer) + acc_list_9layer.append(acc_9layer) + + plt.figure(figsize=(10, 6)) + plt.plot(range(1, EPOCH + 1), acc_list_3layer, label="3-Layer Network", marker="o") + plt.plot(range(1, EPOCH + 1), acc_list_9layer, label="9-Layer Network", marker="s") + plt.xlabel("Epoch") + plt.ylabel("Accuracy On TestSet (%)") + plt.title("Comparison of Network Depths") + plt.legend() + plt.grid(True) + plt.show() diff --git a/13/6.py b/13/6.py new file mode 100644 index 0000000..81993f5 --- /dev/null +++ b/13/6.py @@ -0,0 +1,166 @@ +import matplotlib.pyplot as plt +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.utils.data import DataLoader +from torchvision import datasets, transforms + +# Super parameters +batch_size = 64 +learning_rate = 0.01 +momentum = 0.5 +EPOCH = 10 + +# Prepare dataset +transform = transforms.Compose( + [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))] +) + +train_dataset = datasets.MNIST( + root="./data/mnist", train=True, download=True, transform=transform +) +test_dataset = datasets.MNIST( + root="./data/mnist", train=False, download=True, transform=transform +) +train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True) +test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False) + + +class Net9LayerReLU(torch.nn.Module): + def __init__(self): + super(Net9LayerReLU, self).__init__() + self.fc1 = torch.nn.Linear(28 * 28, 512) + self.fc2 = torch.nn.Linear(512, 512) + self.fc3 = torch.nn.Linear(512, 256) + self.fc4 = torch.nn.Linear(256, 256) + self.fc5 = torch.nn.Linear(256, 128) + self.fc6 = torch.nn.Linear(128, 128) + self.fc7 = torch.nn.Linear(128, 64) + self.fc8 = torch.nn.Linear(64, 64) + self.fc9 = torch.nn.Linear(64, 10) + + def forward(self, x): + x = x.view(-1, 28 * 28) + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = F.relu(self.fc3(x)) + x = F.relu(self.fc4(x)) + x = F.relu(self.fc5(x)) + x = F.relu(self.fc6(x)) + x = F.relu(self.fc7(x)) + x = F.relu(self.fc8(x)) + x = self.fc9(x) + return x + + +class Net9LayerSigmoid(torch.nn.Module): + def __init__(self): + super(Net9LayerSigmoid, self).__init__() + self.fc1 = torch.nn.Linear(28 * 28, 512) + self.fc2 = torch.nn.Linear(512, 512) + self.fc3 = torch.nn.Linear(512, 256) + self.fc4 = torch.nn.Linear(256, 256) + self.fc5 = torch.nn.Linear(256, 128) + self.fc6 = torch.nn.Linear(128, 128) + self.fc7 = torch.nn.Linear(128, 64) + self.fc8 = torch.nn.Linear(64, 64) + self.fc9 = torch.nn.Linear(64, 10) + + def forward(self, x): + x = x.view(-1, 28 * 28) + x = torch.sigmoid(self.fc1(x)) + x = torch.sigmoid(self.fc2(x)) + x = torch.sigmoid(self.fc3(x)) + x = torch.sigmoid(self.fc4(x)) + x = torch.sigmoid(self.fc5(x)) + x = torch.sigmoid(self.fc6(x)) + x = torch.sigmoid(self.fc7(x)) + x = torch.sigmoid(self.fc8(x)) + x = self.fc9(x) + return x + + +# Create two models +model_relu = Net9LayerReLU() +model_sigmoid = Net9LayerSigmoid() + +# Create optimizers +optimizer_relu = torch.optim.SGD( + model_relu.parameters(), lr=learning_rate, momentum=momentum +) +optimizer_sigmoid = torch.optim.SGD( + model_sigmoid.parameters(), lr=learning_rate, momentum=momentum +) + +criterion = torch.nn.CrossEntropyLoss() + + +def train(epoch, model, optimizer, name=""): + model.train() + running_loss = 0.0 + running_total = 0 + running_correct = 0 + + for batch_idx, (inputs, target) in enumerate(train_loader): + optimizer.zero_grad() + outputs = model(inputs) + loss = criterion(outputs, target) + + loss.backward() + optimizer.step() + + running_loss += loss.item() + _, predicted = torch.max(outputs.data, dim=1) + running_total += target.shape[0] + running_correct += (predicted == target).sum().item() + + avg_loss = running_loss / len(train_loader) + avg_acc = 100 * running_correct / running_total + print( + f"[{epoch + 1} / {EPOCH}]: {name} Training Loss: {avg_loss:.3f}, Training Accuracy: {avg_acc:.2f} %" + ) + + +def test(epoch, model, name=""): + model.eval() + correct = 0 + total = 0 + with torch.no_grad(): + for data in test_loader: + images, labels = data + outputs = model(images) + _, predicted = torch.max(outputs.data, dim=1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + + acc = 100 * correct / total + print( + f"[{epoch + 1} / {EPOCH}]: {name} Accuracy on test set after epoch {epoch + 1}: {acc:.1f} %" + ) + return acc + + +if __name__ == "__main__": + acc_list_relu = [] + acc_list_sigmoid = [] + + for epoch in range(EPOCH): + train(epoch, model_relu, optimizer_relu, "ReLU") + train(epoch, model_sigmoid, optimizer_sigmoid, "Sigmoid") + + acc_relu = test(epoch, model_relu, "ReLU") + acc_sigmoid = test(epoch, model_sigmoid, "Sigmoid") + + acc_list_relu.append(acc_relu) + acc_list_sigmoid.append(acc_sigmoid) + + plt.figure(figsize=(10, 6)) + plt.plot(range(1, EPOCH + 1), acc_list_relu, label="ReLU", marker="o") + plt.plot(range(1, EPOCH + 1), acc_list_sigmoid, label="Sigmoid", marker="s") + plt.xlabel("Epoch") + plt.ylabel("Accuracy On TestSet (%)") + plt.title("Comparison of Activation Functions in 9-Layer Network") + plt.legend() + plt.grid(True) + plt.show()