第60日: 事後評価、課金監査、Safetensors形式エクスポートと解体

import math
import torch

@torch.no_grad()
def estimate_loss(model, val_batches):
# Mean cross-entropy over a FIXED list of held-out (x, y) batches, so two checkpoints are
# compared on identical tokens. model(x, y) returns (logits, loss) as in the Day 48 model.
model.eval()
losses = [model(x, y)[1].item() for x, y in val_batches]
return sum(losses) / len(losses)

@torch.no_grad()
def generate_sample(model, idx, max_new_tokens=100, temperature=0.8, top_k=40, block_size=256):
model.eval()
for _ in range(max_new_tokens):
logits, _ = model(idx[:, -block_size:])
logits = logits[:, -1, :] / temperature
kth = torch.topk(logits, top_k).values[:, [-1]]
logits[logits < kth] = float('-inf') # keep only the top_k candidates
idx = torch.cat([idx, torch.multinomial(torch.softmax(logits, dim=-1), 1)], dim=1)
return idx

def has_token_loop(tokens, max_run=8, min_distinct_ratio=0.3):
# Degenerate output: one token repeated more than max_run times in a row,
# or too few distinct tokens overall (e.g. 'the the the ...').
longest = run = 1
for a, b in zip(tokens, tokens[1:]):
run = run + 1 if a == b else 1
longest = max(longest, run)
return longest > max_run or len(set(tokens)) / len(tokens) < min_distinct_ratio

ckpt = torch.load('ckpt.pt', map_location='cpu')
model.load_state_dict(ckpt['model'])
model.eval()

val_loss = estimate_loss(model, val_batches)
ppl = math.exp(val_loss)
print(f'L_val={val_loss:.4f} PPL={ppl:.2f} uniform baseline ln(V)={math.log(vocab_size):.4f}')

prompt = torch.tensor([encode('Once upon a time')])
samples = [generate_sample(model, prompt)[0, prompt.size(1):].tolist() for _ in range(5)]

# Why these thresholds: an untrained model guesses uniformly, so its loss is ln(V).
# For character-level Shakespeare V = 65 and ln(65) = 4.1744, so L_val < 4.0 means the model beats
# guessing; PPL < 50 is the same check for a larger vocabulary (ln(50) = 3.9120).
assert val_loss < math.log(vocab_size)
assert ppl < 50.0
assert not any(has_token_loop(s) for s in samples)
for s in samples:
print(repr(decode(s))) # read them: grammatical fragments, names, line breaks; not one token

# Record: checkpoint step, L_val, PPL, the five samples, temperature=0.8, top_k=40 and the seed.

明細：
計算 = 16.7 時間 * $1.50/時間 = $25.05
ストレージ = 50 GB * $0.10/GB 月 * (1 / 30.4) 月 = $0.164、表示は $0.16
送信 = 2 GB * $0.08/GB = $0.16
合計 = 25.05 + 0.16 + 0.16 = $25.37

Day 58 の上限（$20.00）との照合：
$25.37 > $20.00 なので、照合は FAIL である。予算を $5.37（26.9%）超えている。
ストレージと送信で $0.32 かかるので、収まる最大の計算時間は (20.00 - 0.32) / 1.50 = 13.12 時間。実行は 16.7 時間を使い、3.58 時間多すぎた。正しい事前チェックなら約 13 時間の実時間ガードを設けるか、トークン数を減らして計画したはずである。

効率：
10 億トークンあたりのコスト = $25.37 / 10 = $2.54。
暗黙のスループット = 10e9 トークン / (16.7 * 3600 秒) ≈ 166,000 トークン/秒。

報告の仕方：FAIL と書き、超過額、原因（計画した予算ではなく 16.7 時間かかった）、対策（強制的な時間制限と上限より低い予算アラート）を示す。超過を丸めて消したり、別の項目に移したりしない。

よくある誤り：ストレージを 1 日ではなく 1 か月分（$5.00）請求する。各行を丸めてから足す。上限は計画にすぎないとして $25.37 を予算内とみなす。

import hashlib
import os
import torch
from safetensors.torch import save_file, load_file

ckpt = torch.load('checkpoint.pt', map_location='cpu')
state = ckpt['model'] # keep weights only; 'optimizer', 'step' etc. are dropped

# torch.compile prefixes every key with '_orig_mod.'; strip it so plain models can load the file.
state = {k.removeprefix('_orig_mod.'): v for k, v in state.items()}

# safetensors refuses tensors that share storage (GPT weight tying: lm_head.weight IS
# transformer.wte.weight) and non-contiguous views. Save one independent copy of each tensor;
# re-tie the weights after loading with model.lm_head.weight = model.transformer.wte.weight.
state = {k: v.detach().contiguous().clone() for k, v in state.items()}
save_file(state, 'model.safetensors', metadata={'source': 'checkpoint.pt', 'step': str(ckpt.get('step', ''))})

reloaded = load_file('model.safetensors')
assert set(reloaded) == set(state)
assert all(torch.equal(reloaded[k], state[k]) for k in state) # bit-exact round trip

digest = hashlib.sha256(open('model.safetensors', 'rb').read()).hexdigest()
print(f"{os.path.getsize('model.safetensors') / 2**20:.1f} MiB, sha256={digest}")
# Copy the file home (rsync -av --progress), run sha256sum on the local copy, and compare digests.
# Only then tear down. Why safetensors: torch.load on a .pt file unpickles, and a pickle can run
# arbitrary code; a .safetensors file is a JSON header plus raw tensor bytes, with no code at all.

# Teardown checklist (provider CLI names differ; record the command output, not a screenshot):
# 1. terminate the instance -> instance state 'terminated', not 'stopped'
# 2. delete every attached volume -> volume list for the project is empty
# 3. delete snapshots and buckets you created for this run
# 4. list again an hour later -> active instances = 0, active volumes = 0
# A stopped instance still bills for its disk; an orphaned volume bills until you delete it.

import re

LANGS = ('en', 'ja', 'zh-Hans', 'zh-Hant')
METRICS = ('final_step', 'val_loss', 'ppl', 'mfu', 'total_cost_usd')

# Example report: numbers live once per language section and must be identical everywhere.
# The values below are illustrative; copy yours from the run log and the billing export.
report = {lang: {'final_step': 5000, 'val_loss': 1.4697, 'ppl': 4.3477, 'mfu': 0.3135,
'total_cost_usd': 25.37, 'notes': '...'} for lang in LANGS}
report['teardown_confirmed'] = True
report['teardown_evidence'] = 'instances=0 volumes=0 (provider CLI output, 2026-11-01)'

for lang in LANGS:
for key in METRICS:
assert report[lang][key] == report['en'][key], (lang, key) # same value, every edition
assert report['en']['ppl'] > 0
assert abs(report['en']['ppl'] - 2.718281828 ** report['en']['val_loss']) < 1e-3 # PPL = exp(L_val)
assert 0 < report['en']['mfu'] < 1 # MFU is a fraction, not a percent

# No secrets or private addresses anywhere in the text.
SECRET = re.compile(r'(?:\d{1,3}\.){3}\d{1,3}|AKIA[0-9A-Z]{16}|(?:sk|hf)_[A-Za-z0-9]{16,}|BEGIN [A-Z ]*PRIVATE KEY')
text = repr(report)
assert not SECRET.search(text), SECRET.search(text)

assert report['teardown_confirmed'] is True
assert 'instances=0' in report['teardown_evidence'] and 'volumes=0' in report['teardown_evidence']
print('report parity, secret scan and teardown sign-off passed')

# If the bill exceeded the Day 58 ceiling (25.37 > 20.00 in the Activity 2 example), the report
# says so in every language and names the cause; it does not round the overrun away.