From 6cc8f903ccea424bda2b4fa8fa94131dae27d3e3 Mon Sep 17 00:00:00 2001 From: Luciferian Ink Date: Thu, 13 Jun 2024 18:36:19 -0500 Subject: [PATCH 1/5] restore fixes --- benchmarks/benchmark_optimizer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/benchmark_optimizer.py b/benchmarks/benchmark_optimizer.py index 8f93ff692..b976f3347 100644 --- a/benchmarks/benchmark_optimizer.py +++ b/benchmarks/benchmark_optimizer.py @@ -98,7 +98,7 @@ def run_trainer(batch_size: int, batch_time: float, client_mode: bool, verbose: grad_scaler = hivemind.GradScaler() else: # check that hivemind.Optimizer supports regular PyTorch grad scaler as well - grad_scaler = torch.cuda.amp.GradScaler(enabled=args.use_amp) + grad_scaler = torch.amp.GradScaler(enabled=args.use_amp) prev_time = time.perf_counter() @@ -107,7 +107,7 @@ def run_trainer(batch_size: int, batch_time: float, client_mode: bool, verbose: batch = torch.randint(0, len(X_train), (batch_size,)) - with torch.cuda.amp.autocast() if args.use_amp else nullcontext(): + with torch.amp.autocast() if args.use_amp else nullcontext(): loss = F.cross_entropy(model(X_train[batch].to(args.device)), y_train[batch].to(args.device)) grad_scaler.scale(loss).backward() From ec1d33ce05c9bc7b1e48c9c215d9c0003c39a94b Mon Sep 17 00:00:00 2001 From: Luciferian Ink Date: Sat, 13 Jul 2024 06:09:12 -0500 Subject: [PATCH 2/5] fix benchmarking --- benchmarks/benchmark_optimizer.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/benchmarks/benchmark_optimizer.py b/benchmarks/benchmark_optimizer.py index b976f3347..dc81b4599 100644 --- a/benchmarks/benchmark_optimizer.py +++ b/benchmarks/benchmark_optimizer.py @@ -16,6 +16,15 @@ from hivemind.optim.optimizer import Optimizer from hivemind.utils.crypto import RSAPrivateKey +from packaging import version + +torch_version = torch.__version__.split("+")[0] + +if version.parse(torch_version) >= version.parse("2.3.0"): + from torch.amp import GradScaler +else: + from torch.cuda.amp import GradScaler + @dataclass(frozen=True) class TrainingArguments: @@ -98,7 +107,7 @@ def run_trainer(batch_size: int, batch_time: float, client_mode: bool, verbose: grad_scaler = hivemind.GradScaler() else: # check that hivemind.Optimizer supports regular PyTorch grad scaler as well - grad_scaler = torch.amp.GradScaler(enabled=args.use_amp) + grad_scaler = GradScaler(enabled=args.use_amp) prev_time = time.perf_counter() From f9ae24c371f9d5027e0d589a858ad34dea9d10f7 Mon Sep 17 00:00:00 2001 From: Luciferian Ink Date: Sat, 13 Jul 2024 06:21:53 -0500 Subject: [PATCH 3/5] run isort --- benchmarks/benchmark_optimizer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/benchmarks/benchmark_optimizer.py b/benchmarks/benchmark_optimizer.py index dc81b4599..570d1752a 100644 --- a/benchmarks/benchmark_optimizer.py +++ b/benchmarks/benchmark_optimizer.py @@ -8,6 +8,7 @@ import torch import torchvision +from packaging import version from torch import nn as nn from torch.nn import functional as F from torch.utils.data import Dataset @@ -16,8 +17,6 @@ from hivemind.optim.optimizer import Optimizer from hivemind.utils.crypto import RSAPrivateKey -from packaging import version - torch_version = torch.__version__.split("+")[0] if version.parse(torch_version) >= version.parse("2.3.0"): From 58d95fceaf5b50ece3e640f5ed33b1ddeb76b6ca Mon Sep 17 00:00:00 2001 From: Max Ryabinin Date: Sat, 19 Oct 2024 09:46:20 +0200 Subject: [PATCH 4/5] Use autocast depending on the version --- benchmarks/benchmark_optimizer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/benchmark_optimizer.py b/benchmarks/benchmark_optimizer.py index 570d1752a..3907408bd 100644 --- a/benchmarks/benchmark_optimizer.py +++ b/benchmarks/benchmark_optimizer.py @@ -20,9 +20,9 @@ torch_version = torch.__version__.split("+")[0] if version.parse(torch_version) >= version.parse("2.3.0"): - from torch.amp import GradScaler + from torch.amp import GradScaler, autocast else: - from torch.cuda.amp import GradScaler + from torch.cuda.amp import GradScaler, autocast @dataclass(frozen=True) @@ -115,7 +115,7 @@ def run_trainer(batch_size: int, batch_time: float, client_mode: bool, verbose: batch = torch.randint(0, len(X_train), (batch_size,)) - with torch.amp.autocast() if args.use_amp else nullcontext(): + with autocast() if args.use_amp else nullcontext(): loss = F.cross_entropy(model(X_train[batch].to(args.device)), y_train[batch].to(args.device)) grad_scaler.scale(loss).backward() From 2197f845be169a21fca5d1fab6e66df38abe73d1 Mon Sep 17 00:00:00 2001 From: Max Ryabinin Date: Sat, 14 Dec 2024 15:10:44 +0000 Subject: [PATCH 5/5] Trigger CI