Recall везде + фиксы окружения
- recall_macro/recall_weighted во всех скриптах с метриками (train_simple, train_gpu, autolog_demo, hyperparam_sweep, grid_search_cv, load_and_predict, compare_runs) - grid_search_cv: multi-metric scoring (accuracy + recall_macro) - register_model: выбор лучшей модели по recall (--metric) - MLflow warnings: name вместо artifact_path, сигнатура модели, numpy input_example для pytorch, подавление env-var INFO - start_ui.sh: порт 5555 + file-store backend (фикс: UI не показывал эксперименты, т.к. скрипты писали в mlruns/, а сервер читал sqlite) - setup_server.sh: развилка macOS/Linux + проверка MPS - train_gpu: get_device CUDA→MPS→CPU, num_workers=0 на macOS - AGENTS.md: гид для агента + саморегламент обновления - README/CODE_WALKTHROUGH: выровнена нумерация уроков 8/9 - .gitignore: +.DS_Store - удалён scripts/setup_git.sh
This commit is contained in:
+12
-2
@@ -13,11 +13,15 @@
|
||||
from sklearn.datasets import load_wine
|
||||
from sklearn.ensemble import GradientBoostingClassifier
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.metrics import accuracy_score
|
||||
from sklearn.metrics import accuracy_score, recall_score
|
||||
|
||||
import mlflow
|
||||
import mlflow.sklearn
|
||||
|
||||
# Подавляем INFO MLflow о переменных окружения (напр. OPENAI_API_KEY) при логировании модели
|
||||
import os
|
||||
os.environ.setdefault("MLFLOW_RECORD_ENV_VARS_IN_MODEL_LOGGING", "false")
|
||||
|
||||
|
||||
def main():
|
||||
# ─── КЛЮЧЕВАЯ СТРОКА ──────────────────────────────────────
|
||||
@@ -60,12 +64,18 @@ def main():
|
||||
# Можно добавить и ручные метрики — они дополнят автолог
|
||||
y_pred = model.predict(X_test)
|
||||
acc = accuracy_score(y_test, y_pred)
|
||||
recall_macro = recall_score(y_test, y_pred, average="macro")
|
||||
recall_weighted = recall_score(y_test, y_pred, average="weighted")
|
||||
mlflow.log_metric("manual_accuracy", acc)
|
||||
mlflow.log_metric("manual_recall_macro", recall_macro)
|
||||
mlflow.log_metric("manual_recall_weighted", recall_weighted)
|
||||
|
||||
print(f"\n✅ Готово! Откройте MLflow UI:")
|
||||
print(f" Эксперимент: autolog_wine")
|
||||
print(f" Run: {run.info.run_id}")
|
||||
print(f" Accuracy: {acc:.4f}")
|
||||
print(f" Accuracy: {acc:.4f}")
|
||||
print(f" Recall (macro): {recall_macro:.4f}")
|
||||
print(f" Recall (weighted):{recall_weighted:.4f}")
|
||||
print(f"\n🔍 Что autolog залогировал автоматически:")
|
||||
print(f" • Параметры: n_estimators, learning_rate, max_depth, ...")
|
||||
print(f" • Метрики: training_accuracy, training_log_loss, ...")
|
||||
|
||||
+21
-5
@@ -35,19 +35,35 @@ def main():
|
||||
print("❌ Нет запусков!")
|
||||
return
|
||||
|
||||
# Какие метрики показывать (показываем recall везде, где он есть)
|
||||
metric_cols = [args.metric, "recall_macro", "recall_weighted",
|
||||
"test_recall_macro", "test_recall_weighted",
|
||||
"manual_recall_macro"]
|
||||
|
||||
print(f"📊 Топ-{len(runs)} запусков в '{args.experiment}' по {args.metric}:")
|
||||
print(f"{'#':>3} | {'Run ID':>36} | {'accuracy':>9} | {'n_est':>6} | {'depth':>5}")
|
||||
print("-" * 75)
|
||||
header = f"{'#':>3} | {'Run ID':>20} | {'n_est':>6} | {'depth':>5}"
|
||||
for m in metric_cols:
|
||||
header += f" | {m[:14]:>14}"
|
||||
print(header)
|
||||
print("-" * len(header))
|
||||
|
||||
for i, run in enumerate(runs):
|
||||
run_id = run.info.run_id
|
||||
acc = run.data.metrics.get(args.metric, 0)
|
||||
n_est = run.data.params.get("n_estimators", "—")
|
||||
depth = run.data.params.get("max_depth", "—")
|
||||
print(f"{i+1:>3} | {run_id:>36} | {acc:>9.4f} | {n_est:>6} | {depth:>5}")
|
||||
row = f"{i+1:>3} | {run_id[:20]:>20} | {str(n_est):>6} | {str(depth):>5}"
|
||||
for m in metric_cols:
|
||||
v = run.data.metrics.get(m)
|
||||
row += f" | {v:>14.4f}" if v is not None else f" | {'—':>14}"
|
||||
print(row)
|
||||
|
||||
best = runs[0]
|
||||
print(f"\n🏆 Лучший: accuracy={best.data.metrics.get(args.metric, 0):.4f}")
|
||||
print(f"\n🏆 Лучший по {args.metric}: {best.data.metrics.get(args.metric, 0):.4f}")
|
||||
# Покажем recall лучшего, если он залогирован
|
||||
for rm in ("recall_macro", "test_recall_macro", "manual_recall_macro"):
|
||||
rv = best.data.metrics.get(rm)
|
||||
if rv is not None:
|
||||
print(f" {rm} = {rv:.4f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+19
-5
@@ -10,11 +10,15 @@ MLflow autolog автоматически залогирует КАЖУЮ поп
|
||||
from sklearn.datasets import load_digits
|
||||
from sklearn.ensemble import RandomForestClassifier
|
||||
from sklearn.model_selection import GridSearchCV, train_test_split
|
||||
from sklearn.metrics import accuracy_score
|
||||
from sklearn.metrics import accuracy_score, recall_score
|
||||
|
||||
import mlflow
|
||||
import mlflow.sklearn
|
||||
|
||||
# Подавляем INFO MLflow о переменных окружения (напр. OPENAI_API_KEY) при логировании модели
|
||||
import os
|
||||
os.environ.setdefault("MLFLOW_RECORD_ENV_VARS_IN_MODEL_LOGGING", "false")
|
||||
|
||||
|
||||
def main():
|
||||
# autolog для sklearn — залогирует все промежуточные попытки GridSearch
|
||||
@@ -56,7 +60,8 @@ def main():
|
||||
estimator=RandomForestClassifier(random_state=42, n_jobs=-1),
|
||||
param_grid=param_grid,
|
||||
cv=5,
|
||||
scoring="accuracy",
|
||||
scoring=["accuracy", "recall_macro"], # мульти-метрика: accuracy + recall
|
||||
refit="accuracy", # лучший выбираем по accuracy
|
||||
n_jobs=-1,
|
||||
verbose=1,
|
||||
)
|
||||
@@ -66,14 +71,23 @@ def main():
|
||||
best = grid.best_estimator_
|
||||
y_pred = best.predict(X_test)
|
||||
test_acc = accuracy_score(y_test, y_pred)
|
||||
test_recall_macro = recall_score(y_test, y_pred, average="macro")
|
||||
test_recall_weighted = recall_score(y_test, y_pred, average="weighted")
|
||||
best_cv_recall = grid.cv_results_["mean_test_recall_macro"][grid.best_index_]
|
||||
|
||||
mlflow.log_param("best_params", str(grid.best_params_))
|
||||
mlflow.log_metric("best_cv_score", grid.best_score_)
|
||||
mlflow.log_metric("best_cv_score", grid.best_score_) # accuracy (refit)
|
||||
mlflow.log_metric("best_cv_recall_macro", best_cv_recall)
|
||||
mlflow.log_metric("test_accuracy", test_acc)
|
||||
mlflow.log_metric("test_recall_macro", test_recall_macro)
|
||||
mlflow.log_metric("test_recall_weighted", test_recall_weighted)
|
||||
|
||||
print(f"\n🏆 Лучшие параметры: {grid.best_params_}")
|
||||
print(f" CV score: {grid.best_score_:.4f}")
|
||||
print(f" Test accuracy: {test_acc:.4f}")
|
||||
print(f" CV accuracy: {grid.best_score_:.4f}")
|
||||
print(f" CV recall_macro: {best_cv_recall:.4f}")
|
||||
print(f" Test accuracy: {test_acc:.4f}")
|
||||
print(f" Test recall_macro: {test_recall_macro:.4f}")
|
||||
print(f" Test recall_weighted: {test_recall_weighted:.4f}")
|
||||
print(f"\n📊 В MLflow UI:")
|
||||
print(f" Эксперимент: gridsearch_digits")
|
||||
print(f" Parent run: {run.info.run_id}")
|
||||
|
||||
@@ -104,6 +104,7 @@ def main():
|
||||
y_pred = model.predict(X_test)
|
||||
test_acc = accuracy_score(y_test, y_pred)
|
||||
test_recall = recall_score(y_test, y_pred, average="macro")
|
||||
test_recall_weighted = recall_score(y_test, y_pred, average="weighted")
|
||||
test_precision = precision_score(y_test, y_pred, average="macro")
|
||||
test_f1 = f1_score(y_test, y_pred, average="macro")
|
||||
|
||||
@@ -112,6 +113,7 @@ def main():
|
||||
mlflow.log_metric("cv_std_accuracy", cv_scores.std())
|
||||
mlflow.log_metric("test_accuracy", test_acc)
|
||||
mlflow.log_metric("test_recall_macro", test_recall)
|
||||
mlflow.log_metric("test_recall_weighted", test_recall_weighted)
|
||||
mlflow.log_metric("test_precision_macro", test_precision)
|
||||
mlflow.log_metric("test_f1_macro", test_f1)
|
||||
|
||||
@@ -129,7 +131,7 @@ def main():
|
||||
filled = int(bar_len * (i + 1) / len(combos))
|
||||
bar = "█" * filled + "░" * (bar_len - filled)
|
||||
print(f"\r [{bar}] {i+1}/{len(combos)} | "
|
||||
f"acc={test_acc:.4f} | {run_name[:40]:<40}", end="")
|
||||
f"acc={test_acc:.4f} rec={test_recall:.4f} | {run_name[:38]:<38}", end="")
|
||||
|
||||
print(f"\n\n{'='*60}")
|
||||
print(f"🏆 Лучший результат:")
|
||||
|
||||
+18
-3
@@ -29,10 +29,25 @@ def main():
|
||||
print(" Сначала запустите train_simple.py и register_model.py")
|
||||
return
|
||||
|
||||
# Делаем предсказание на нескольких образцах
|
||||
# --- Качество на отложенной тестовой выборке (как в train_simple.py) ---
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.metrics import accuracy_score, recall_score
|
||||
|
||||
digits = load_digits()
|
||||
X, y = digits.data, digits.target
|
||||
X_train, X_test, y_train, y_test = train_test_split(
|
||||
X, y, test_size=0.2, random_state=42
|
||||
)
|
||||
y_pred = model.predict(X_test)
|
||||
acc = accuracy_score(y_test, y_pred)
|
||||
recall_macro = recall_score(y_test, y_pred, average="macro")
|
||||
recall_weighted = recall_score(y_test, y_pred, average="weighted")
|
||||
print(f"\n📈 Качество загруженной модели на тесте ({len(y_test)} образов):")
|
||||
print(f" Accuracy: {acc:.4f}")
|
||||
print(f" Recall (macro): {recall_macro:.4f}")
|
||||
print(f" Recall (weighted): {recall_weighted:.4f}")
|
||||
|
||||
# --- Демо: предсказание на нескольких случайных образцах ---
|
||||
n_samples = 5
|
||||
indices = np.random.choice(len(X), n_samples, replace=False)
|
||||
samples = X[indices]
|
||||
@@ -46,8 +61,8 @@ def main():
|
||||
ok = "✅" if predictions[i] == true_labels[i] else "❌"
|
||||
print(f"{i+1:>6} | {true_labels[i]:>8} | {predictions[i]:>14} | {ok}")
|
||||
|
||||
acc = np.mean(predictions == true_labels)
|
||||
print(f"\nТочность на {n_samples} образцах: {acc:.2%}")
|
||||
demo_acc = np.mean(predictions == true_labels)
|
||||
print(f"\nТочность на {n_samples} образцах: {demo_acc:.2%}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
"""
|
||||
Урок 3: Model Registry — управление версиями моделей
|
||||
=====================================================
|
||||
Ищет лучший run по метрике accuracy, регистрирует модель
|
||||
в Model Registry и переводит её в стадию Production.
|
||||
Ищет лучший run по заданной метрике (по умолчанию accuracy),
|
||||
регистрирует модель в Model Registry и переводит её в стадию Production.
|
||||
|
||||
Можно выбирать лучшую модель по recall:
|
||||
python src/register_model.py --experiment digits_classification --metric recall_macro
|
||||
python src/register_model.py --experiment sweep_digits --metric test_recall_macro
|
||||
|
||||
Запуск:
|
||||
python src/register_model.py --experiment digits_classification
|
||||
|
||||
+59
-10
@@ -2,11 +2,15 @@
|
||||
Урок 2: MLflow + PyTorch на GPU (RTX 3090)
|
||||
=============================================
|
||||
Обучаем CNN на MNIST. Автоматически использует CUDA если доступна,
|
||||
иначе — CPU. Все параметры, метрики и модель логируются в MLflow.
|
||||
иначе — CPU. На Apple Silicon используется MPS.
|
||||
Все параметры, метрики и модель логируются в MLflow.
|
||||
|
||||
Запуск на 3090:
|
||||
Запуск на 3090 (CUDA):
|
||||
python src/train_gpu.py --epochs 10 --batch-size 256 --lr 0.001
|
||||
|
||||
Запуск на Apple Silicon (MPS):
|
||||
python src/train_gpu.py --epochs 5 --batch-size 128 --lr 0.001
|
||||
|
||||
Запуск на CPU (для теста):
|
||||
python src/train_gpu.py --epochs 2 --batch-size 64 --lr 0.01
|
||||
"""
|
||||
@@ -17,15 +21,21 @@ import time
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.optim as optim
|
||||
from torch.utils.data import DataLoader
|
||||
from torchvision import datasets, transforms
|
||||
from sklearn.metrics import recall_score
|
||||
|
||||
import mlflow
|
||||
import mlflow.pytorch
|
||||
|
||||
# Подавляем INFO MLflow о переменных окружения (напр. OPENAI_API_KEY) при логировании модели
|
||||
import os
|
||||
os.environ.setdefault("MLFLOW_RECORD_ENV_VARS_IN_MODEL_LOGGING", "false")
|
||||
|
||||
|
||||
# ─── Модель: простая CNN ───
|
||||
class SimpleCNN(nn.Module):
|
||||
@@ -52,6 +62,15 @@ class SimpleCNN(nn.Module):
|
||||
return self.classifier(x)
|
||||
|
||||
|
||||
def get_device() -> torch.device:
|
||||
"""Выбор устройства: CUDA -> MPS (Apple Silicon) -> CPU."""
|
||||
if torch.cuda.is_available():
|
||||
return torch.device("cuda")
|
||||
if getattr(torch.backends, "mps", None) is not None and torch.backends.mps.is_available():
|
||||
return torch.device("mps")
|
||||
return torch.device("cpu")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="MLflow + PyTorch GPU demo")
|
||||
parser.add_argument("--epochs", type=int, default=5)
|
||||
@@ -60,13 +79,16 @@ def main():
|
||||
parser.add_argument("--experiment-name", type=str, default="mnist_cnn_gpu")
|
||||
args = parser.parse_args()
|
||||
|
||||
# ─── Устройство ───
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
# ─── Устройство: CUDA -> MPS -> CPU ───
|
||||
device = get_device()
|
||||
print(f"🖥️ Устройство: {device}")
|
||||
gpu_name = None
|
||||
if device.type == "cuda":
|
||||
gpu_name = torch.cuda.get_device_name(0)
|
||||
gpu_mem = torch.cuda.get_device_properties(0).total_memory / 1e9
|
||||
print(f" GPU: {gpu_name} ({gpu_mem:.1f} GB)")
|
||||
elif device.type == "mps":
|
||||
print(" Apple Silicon GPU (MPS)")
|
||||
|
||||
# ─── MLflow эксперимент ───
|
||||
mlflow.set_experiment(args.experiment_name)
|
||||
@@ -80,9 +102,22 @@ def main():
|
||||
os.makedirs(data_dir, exist_ok=True)
|
||||
train_ds = datasets.MNIST(data_dir, train=True, download=True, transform=transform)
|
||||
test_ds = datasets.MNIST(data_dir, train=False, download=True, transform=transform)
|
||||
train_loader = DataLoader(train_ds, batch_size=args.batch_size, shuffle=True, num_workers=4)
|
||||
test_loader = DataLoader(test_ds, batch_size=args.batch_size, shuffle=False, num_workers=4)
|
||||
print(f"📊 Train: {len(train_ds)}, Test: {len(test_ds)}")
|
||||
|
||||
# num_workers: на macOS с MPS многопроцессная загрузка нестабильна
|
||||
# (fork + MPS) → используем 0 воркеров на mac, 4 на Linux/CUDA.
|
||||
import platform
|
||||
num_workers = 0 if platform.system() == "Darwin" else 4
|
||||
pin_memory = device.type == "cuda"
|
||||
train_loader = DataLoader(
|
||||
train_ds, batch_size=args.batch_size, shuffle=True,
|
||||
num_workers=num_workers, pin_memory=pin_memory,
|
||||
)
|
||||
test_loader = DataLoader(
|
||||
test_ds, batch_size=args.batch_size, shuffle=False,
|
||||
num_workers=num_workers, pin_memory=pin_memory,
|
||||
)
|
||||
print(f"📊 Train: {len(train_ds)}, Test: {len(test_ds)} "
|
||||
f"(num_workers={num_workers}, pin_memory={pin_memory})")
|
||||
|
||||
# ─── MLflow run ───
|
||||
with mlflow.start_run(run_name=f"cnn_e{args.epochs}_bs{args.batch_size}") as run:
|
||||
@@ -130,6 +165,7 @@ def main():
|
||||
# ─── Валидация ───
|
||||
model.eval()
|
||||
correct, total = 0, 0
|
||||
all_preds, all_targets = [], []
|
||||
with torch.no_grad():
|
||||
for data, target in test_loader:
|
||||
data, target = data.to(device), target.to(device)
|
||||
@@ -137,18 +173,28 @@ def main():
|
||||
pred = output.argmax(dim=1)
|
||||
correct += (pred == target).sum().item()
|
||||
total += target.size(0)
|
||||
all_preds.append(pred.cpu())
|
||||
all_targets.append(target.cpu())
|
||||
|
||||
acc = correct / total
|
||||
# Recall (macro) — полнота по всем классам MNIST
|
||||
recall_macro = recall_score(
|
||||
torch.cat(all_targets).numpy(),
|
||||
torch.cat(all_preds).numpy(),
|
||||
average="macro",
|
||||
zero_division=0,
|
||||
)
|
||||
test_accs.append(acc)
|
||||
elapsed = time.time() - t0
|
||||
|
||||
# Логируем метрики по эпохам
|
||||
mlflow.log_metric("train_loss", avg_loss, step=epoch)
|
||||
mlflow.log_metric("test_accuracy", acc, step=epoch)
|
||||
mlflow.log_metric("test_recall_macro", recall_macro, step=epoch)
|
||||
mlflow.log_metric("epoch_time_sec", elapsed, step=epoch)
|
||||
|
||||
print(f" → Epoch {epoch+1}: loss={avg_loss:.4f}, "
|
||||
f"acc={acc:.4f}, time={elapsed:.1f}s")
|
||||
f"acc={acc:.4f}, recall={recall_macro:.4f}, time={elapsed:.1f}s")
|
||||
|
||||
# ─── График обучения ───
|
||||
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))
|
||||
@@ -166,17 +212,20 @@ def main():
|
||||
plt.close(fig)
|
||||
|
||||
# ─── Логируем модель ───
|
||||
# input_example — numpy ndarray (MLflow 3.x не принимает torch.Tensor).
|
||||
# Форма как у входа модели: (batch, channels, H, W) = (1, 1, 28, 28).
|
||||
mlflow.pytorch.log_model(
|
||||
model,
|
||||
artifact_path="model",
|
||||
name="model", # name вместо устаревшего artifact_path
|
||||
registered_model_name=None,
|
||||
input_example=torch.randn(1, 1, 28, 28).to(device),
|
||||
input_example=np.random.rand(1, 1, 28, 28).astype(np.float32),
|
||||
)
|
||||
|
||||
# ─── Теги ───
|
||||
mlflow.set_tag("model_type", "SimpleCNN")
|
||||
mlflow.set_tag("framework", "PyTorch")
|
||||
mlflow.set_tag("dataset", "MNIST")
|
||||
mlflow.set_tag("device_type", device.type)
|
||||
|
||||
final_acc = test_accs[-1]
|
||||
print(f"\n✅ Обучение завершено!")
|
||||
|
||||
+19
-5
@@ -15,12 +15,17 @@ import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
from sklearn.datasets import load_digits
|
||||
from sklearn.ensemble import RandomForestClassifier
|
||||
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report
|
||||
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report, recall_score
|
||||
from sklearn.model_selection import train_test_split
|
||||
|
||||
import mlflow
|
||||
import mlflow.sklearn
|
||||
|
||||
# Подавляем INFO MLflow о переменных окружения (напр. OPENAI_API_KEY) при логировании модели
|
||||
import os
|
||||
os.environ.setdefault("MLFLOW_RECORD_ENV_VARS_IN_MODEL_LOGGING", "false")
|
||||
from mlflow.models.signature import infer_signature
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="MLflow + scikit-learn demo")
|
||||
@@ -62,10 +67,16 @@ def main():
|
||||
# --- Предсказание и метрики ---
|
||||
y_pred = model.predict(X_test)
|
||||
acc = accuracy_score(y_test, y_pred)
|
||||
print(f"Accuracy: {acc:.4f}")
|
||||
recall_macro = recall_score(y_test, y_pred, average="macro")
|
||||
recall_weighted = recall_score(y_test, y_pred, average="weighted")
|
||||
print(f"Accuracy: {acc:.4f}")
|
||||
print(f"Recall (macro): {recall_macro:.4f}")
|
||||
print(f"Recall (weighted): {recall_weighted:.4f}")
|
||||
|
||||
# Логируем метрики
|
||||
mlflow.log_metric("accuracy", acc)
|
||||
mlflow.log_metric("recall_macro", recall_macro)
|
||||
mlflow.log_metric("recall_weighted", recall_weighted)
|
||||
# можно логировать несколько шагов (для графиков в UI)
|
||||
for i, tree in enumerate(model.estimators_):
|
||||
tree_acc = accuracy_score(y_test, tree.predict(X_test))
|
||||
@@ -92,11 +103,14 @@ def main():
|
||||
f.write(report)
|
||||
mlflow.log_artifact(report_path)
|
||||
|
||||
# --- Логируем саму модель ---
|
||||
# --- Логируем саму модель (с сигнатурой и примером входа) ---
|
||||
signature = infer_signature(X_test[:5], model.predict(X_test[:5]))
|
||||
mlflow.sklearn.log_model(
|
||||
model,
|
||||
artifact_path="model",
|
||||
registered_model_name=None, # регистрация — в отдельном скрипте
|
||||
name="model", # name вместо устаревшего artifact_path
|
||||
signature=signature, # явная сигнатура модели
|
||||
input_example=X_test[:5], # пример входа → убирает warning
|
||||
registered_model_name=None, # регистрация — в отдельном скрипте
|
||||
)
|
||||
|
||||
# --- Теги ---
|
||||
|
||||
Reference in New Issue
Block a user