{"id":2476,"date":"2026-09-15T01:04:13","date_gmt":"2026-09-14T17:04:13","guid":{"rendered":"https:\/\/blog.zjml.life\/?p=2476"},"modified":"2026-09-15T01:04:28","modified_gmt":"2026-09-14T17:04:28","slug":"%e8%b7%9f%e7%9d%80-andrej-karpathy-%e9%9b%b6%e5%9f%ba%e7%a1%80%e5%ad%a6%e4%b9%a0%e7%a5%9e%e7%bb%8f%e7%bd%91%e7%bb%9c-lets-build-gpt-%e5%ad%a6%e4%b9%a0%e7%ac%94%e8%ae%b0","status":"publish","type":"post","link":"https:\/\/blog.zjml.life\/?p=2476","title":{"rendered":"\u8ddf\u7740 Andrej Karpathy \u96f6\u57fa\u7840\u5b66\u4e60\u795e\u7ecf\u7f51\u7edc Let’s build GPT \u5b66\u4e60\u7b14\u8bb0"},"content":{"rendered":"\n

\u8fd9\u662fAndrej Karpathy \u96f6\u57fa\u7840\u5b66\u4e60\u795e\u7ecf\u7f51\u7edc\u7684\u89c6\u9891\u8bfe\u7684\u7b2c\u4e03\u8282\u8bfe\u3002\u672c\u6b21\u6211\u4eec\u5c06\u4f1a\u5b66\u4e60\u5982\u4f55\u642d\u5efa\u4e00\u4e2aTransformer\u67b6\u6784\u7684\u8bed\u8a00\u6a21\u578b\u3002<\/p>\n\n\n\n

\u51c6\u5907\u5de5\u4f5c<\/h2>\n\n\n\n

\u5b89\u88c5Python
\u5b89\u88c5Pytorch
\u89c2\u770b\u89c6\u9891\u8bfe\u7a0b<\/a><\/p>\n\n\n\n

\u6b63\u5f0f\u5f00\u59cb\u7f16\u5199\u4ee3\u7801<\/h2><\/div>\n\n\n\n
\n

\u5f15\u5165\u5fc5\u8981\u5305\u548c\u56fa\u5b9a\u968f\u673a\u79cd\u5b50<\/h3><\/div>\n\n\n\n

\u7ecf\u5178\u7684\u7b2c\u4e00\u6b65\uff0c\u5f15\u5165\u5fc5\u987b\u4f7f\u7528\u7684\u5305\uff0c\u5e76\u4e14\u56fa\u5b9a\u968f\u673a\u79cd\u5b50\uff1a<\/p>\n\n\n\n

import torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\ntorch.manual_seed(1337)<\/code><\/pre>\n<\/div><\/div>\n\n\n\n
\n
\n
\u5f15\u5165\u6587\u4ef6\u5e76\u63d0\u53d6\u51fa\u5b57\u7b26<\/h3><\/div>\n\n\n\n
\u5f15\u5165\u6211\u4eec\u7528\u4e8e\u8bad\u7ec3\u7684\u838e\u58eb\u6bd4\u4e9a\u6587\u672c\u96c6\u6587\u4ef6\uff0c\u5e76\u4e14\u5c06\u6587\u4ef6\u4e2d\u51fa\u73b0\u7684\u6240\u6709\u5b57\u7b26\u63d0\u53d6\u51fa\u6765\uff1a<\/p>\n\n\n\n
with open('input.txt', 'r', encoding='utf-8') as f:\n    text = f.read()\nchars = sorted(list(set(text)))\nvocab_size = len(chars)\nprint(''.join(chars))\nprint(vocab_size)<\/code><\/pre>\n<\/div><\/div>\n\n\n\n
\u8f93\u51fa\uff1a<\/p>\n\n\n\n
 !$&',-.3:;?ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz\n65<\/code><\/pre>\n\n\n\n
\u53ef\u4ee5\u770b\u5230\uff0c\u4e00\u5171\u670965\u79cd\u4e0d\u540c\u7684\u5b57\u7b26\u3002<\/p>\n<\/div><\/div>\n\n\n\n
\n
\n
\u7f16\u5199\u7f16\u7801\u5668\u3001\u89e3\u7801\u5668<\/h3><\/div>\n\n\n\n
\u63a5\u4e0b\u6765\u6211\u4eec\u628a\u7f16\u7801\u5668\u548c\u89e3\u7801\u5668\u5199\u51fa\u6765\uff1a<\/p>\n\n\n\n
stoi = { ch:i for i,ch in enumerate(chars) }\nitos = {i:ch for i,ch in enumerate(chars)}\nencode = lambda s: [stoi[c] for c in s]\ndecode = lambda l: ''.join([itos[i] for i in l])\n\nprint(encode(\"hii there\"))\nprint(decode(encode(\"hii there\")))<\/code><\/pre>\n<\/div><\/div>\n\n\n\n
\u8f93\u51fa\uff1a<\/p>\n\n\n\n
[46, 47, 47, 1, 58, 46, 43, 56, 43]\nhii there<\/code><\/pre>\n\n\n\n
\u53ef\u89c1\uff0c\u6211\u4eec\u7684\u7f16\u7801\u5668\u548c\u89e3\u7801\u5668\u5df2\u7ecf\u6b63\u5e38\u5de5\u4f5c\u4e86\u3002<\/p>\n<\/div><\/div>\n\n\n\n
\n
\n
\u5c06\u6587\u672c\u8f6c\u5316\u4e3a\u4e00\u7ef4\u5411\u91cf\uff0c\u5e76\u5207\u5272\u8bad\u7ec3\u96c6\u548c\u9a8c\u8bc1\u96c6<\/h3><\/div>\n\n\n\n
\u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u5c06\u6574\u4e2a\u838e\u58eb\u6bd4\u4e9a\u6587\u672c\u96c6\u8f6c\u5316\u4e3a\u4e00\u7ef4\u5411\u91cf\uff0c\u5e76\u4e14\u5c06\u5176\u63099:1\u7684\u6bd4\u4f8b\u5212\u5206\u4e3a\u8bad\u7ec3\u96c6\u548c\u9a8c\u8bc1\u96c6\uff1a<\/p>\n\n\n\n
# \u5c06\u6570\u636e\u8f6c\u4e3a\u4e00\u7ef4\u5f20\u91cf\ndata = torch.tensor(encode(text), dtype=torch.long)\nprint(data.shape, data.dtype)\nprint(data[:1000])\n\n# \u5c06\u6570\u636e\u5206\u4e3a\u8bad\u7ec3\u96c6\u548c\u9a8c\u8bc1\u96c6\nn = int(0.9*len(data))\ntrain_data = data[:n]\nval_data = data[n:]<\/code><\/pre>\n<\/div><\/div>\n\n\n\n
\u8f93\u51fa\uff1a<\/p>\n\n\n\n
torch.Size([1115394]) torch.int64\ntensor([18, 47, 56, 57, 58,  1, 15, 47, 58, 47, 64, 43, 52, 10,  0, 14, 43, 44,\n        53, 56, 43,  1, 61, 43,  1, 54, 56, 53, 41, 43, 43, 42,  1, 39, 52, 63,\n         1, 44, 59, 56, 58, 46, 43, 56,  6,  1, 46, 43, 39, 56,  1, 51, 43,  1,...<\/code><\/pre>\n<\/div><\/div>\n\n\n\n
\n
\n
\u52a0\u5de5\u8bad\u7ec3\u6837\u672c<\/h3><\/div>\n\n\n\n
\u5bf9\u4e8e\u6211\u4eec\u8bad\u7ec3\u6a21\u578b\uff0c\u6211\u4eec\u63d0\u4f9b\u7684\u6837\u672c\u662f\u8fd9\u6837\u7684\uff1a<\/p>\n\n\n\n
block_size = 8\nprint(train_data[:block_size+1])\n\nx = train_data[:block_size]\ny = train_data[1:block_size+1]\nfor t in range(block_size):\n    context = x[:t+1]\n    target = y[t]\n    print(context)\n    print(target)<\/code><\/pre>\n<\/div><\/div>\n\n\n\n
\u8f93\u51fa\uff1a<\/p>\n\n\n\n
tensor([18, 47, 56, 57, 58,  1, 15, 47, 58])\ntensor([18])\ntensor(47)\ntensor([18, 47])\ntensor(56)\ntensor([18, 47, 56])\ntensor(57)\ntensor([18, 47, 56, 57])\ntensor(58)\ntensor([18, 47, 56, 57, 58])\ntensor(1)\ntensor([18, 47, 56, 57, 58,  1])\ntensor(15)\ntensor([18, 47, 56, 57, 58,  1, 15])\ntensor(47)\ntensor([18, 47, 56, 57, 58,  1, 15, 47])\ntensor(58)<\/code><\/pre>\n\n\n\n
\u6a21\u578b\u5b66\u4e60[18]\u4e4b\u540e\u8f93\u51fa47\uff0c\u5b66\u4e60[18, 47]\u4e4b\u540e\u8f93\u51fa56\uff0c\u4ee5\u6b64\u7c7b\u63a8\u3002<\/p>\n<\/div><\/div>\n\n\n\n
\u6839\u636e\u8fd9\u4e2a\u601d\u60f3\uff0c\u6211\u4eec\u53ef\u4ee5\u8ba9GPU\u5e76\u884c\u7684\u5904\u7406\u5e76\u5b66\u4e60\uff0c\u6bd4\u5982\u63a5\u4e0b\u6765\u6211\u4eec\u53ef\u4ee5\u7f16\u5199\u4ee3\u7801\uff0c\u540c\u65f6\u51c6\u59074\u7ec4\u8bad\u7ec3\u6837\u672c\uff1a<\/p>\n\n\n\n
batch_size = 4 # \u6279\u6b21\u5927\u5c0f\nblock_size = 8 # \u5757\u5927\u5c0f\ndef get_batch(split):\n    data = train_data if split == 'train' else val_data # \u533a\u5206\u52a0\u8f7d\u8bad\u7ec3\u96c6\u8fd8\u662f\u9a8c\u8bc1\u96c6\n    ix = torch.randint(len(data) - block_size,(batch_size,)) # \u968f\u673a\u8d77\u70b9\n    x = torch.stack([data[i:i+block_size] for i in ix])\n    y = torch.stack([data[i+1:i+block_size+1] for i in ix])\n    return x,y\n\nxb,yb = get_batch('train')\nprint('input:')\nprint(xb.shape)\nprint(xb)\nprint('output:')\nprint(yb.shape)\nprint(yb)\n\nprint('------')\n\nfor b in range(batch_size):\n    for t in range(block_size):\n        context = xb[b, :t+1]\n        target = yb[b,t]\n        print(f'context: {context}, target: {target}')<\/code><\/pre>\n\n\n\n
\u8f93\u51fa\uff1a<\/p>\n\n\n\n
input:\ntorch.Size([4, 8])\ntensor([[24, 43, 58,  5, 57,  1, 46, 43],\n        [44, 53, 56,  1, 58, 46, 39, 58],\n        [52, 58,  1, 58, 46, 39, 58,  1],\n        [25, 17, 27, 10,  0, 21,  1, 54]])\noutput:\ntorch.Size([4, 8])\ntensor([[43, 58,  5, 57,  1, 46, 43, 39],\n        [53, 56,  1, 58, 46, 39, 58,  1],\n        [58,  1, 58, 46, 39, 58,  1, 46],\n        [17, 27, 10,  0, 21,  1, 54, 39]])\n------\ncontext: tensor([24]), target: 43\ncontext: tensor([24, 43]), target: 58\ncontext: tensor([24, 43, 58]), target: 5\ncontext: tensor([24, 43, 58,  5]), target: 57\ncontext: tensor([24, 43, 58,  5, 57]), target: 1\ncontext: tensor([24, 43, 58,  5, 57,  1]), target: 46\ncontext: tensor([24, 43, 58,  5, 57,  1, 46]), target: 43\ncontext: tensor([24, 43, 58,  5, 57,  1, 46, 43]), target: 39\ncontext: tensor([44]), target: 53\ncontext: tensor([44, 53]), target: 56\ncontext: tensor([44, 53, 56]), target: 1\ncontext: tensor([44, 53, 56,  1]), target: 58\ncontext: tensor([44, 53, 56,  1, 58]), target: 46\ncontext: tensor([44, 53, 56,  1, 58, 46]), target: 39\ncontext: tensor([44, 53, 56,  1, 58, 46, 39]), target: 58\ncontext: tensor([44, 53, 56,  1, 58, 46, 39, 58]), target: 1\ncontext: tensor([52]), target: 58\ncontext: tensor([52, 58]), target: 1\ncontext: tensor([52, 58,  1]), target: 58\ncontext: tensor([52, 58,  1, 58]), target: 46\ncontext: tensor([52, 58,  1, 58, 46]), target: 39\ncontext: tensor([52, 58,  1, 58, 46, 39]), target: 58\ncontext: tensor([52, 58,  1, 58, 46, 39, 58]), target: 1\ncontext: tensor([52, 58,  1, 58, 46, 39, 58,  1]), target: 46\ncontext: tensor([25]), target: 17\ncontext: tensor([25, 17]), target: 27\ncontext: tensor([25, 17, 27]), target: 10\ncontext: tensor([25, 17, 27, 10]), target: 0\ncontext: tensor([25, 17, 27, 10,  0]), target: 21\ncontext: tensor([25, 17, 27, 10,  0, 21]), target: 1\ncontext: tensor([25, 17, 27, 10,  0, 21,  1]), target: 54\ncontext: tensor([25, 17, 27, 10,  0, 21,  1, 54]), target: 39<\/code><\/pre>\n\n\n\n
\u5b9e\u73b0\u4e8c\u5143\u8bed\u8a00\u6a21\u578b<\/h3><\/div>\n\n\n\n
\u8fd9\u4e2a\u5730\u65b9\uff0c\u6211\u4e5f\u7406\u89e3\u4e86\u5f88\u4e45\uff0c\u6211\u5148\u628a\u4ee3\u7801\u8d34\u4e0a\u6765\uff1a<\/p>\n\n\n\n
class BigramLanguageModel(nn.Module):\n\n    def __init__(self, vocab_size):\n        super().__init__()\n        self.token_embedding_table = nn.Embedding(vocab_size, vocab_size)\n\n    def forward(self, idx, targets):\n\n        logits = self.token_embedding_table(idx)\n\n        return logits\n\nm = BigramLanguageModel(vocab_size)\nout = m(xb,yb)\nprint(out.shape)<\/code><\/pre>\n\n\n\n
\u8f93\u51fa\uff1a<\/p>\n\n\n\n
torch.Size([4, 8, 65])<\/code><\/pre>\n\n\n\n
\u89e3\u91ca\u4e00\u4e0b\uff0c\u9996\u5148\u770b\u4e00\u4e0bxb\u7684\u6837\u5b50\uff1a<\/p>\n\n\n\n
tensor([[24, 43, 58,  5, 57,  1, 46, 43],\n        [44, 53, 56,  1, 58, 46, 39, 58],\n        [52, 58,  1, 58, 46, 39, 58,  1],\n        [25, 17, 27, 10,  0, 21,  1, 54]])<\/code><\/pre>\n\n\n\n
\u53ef\u4ee5\u770b\u5230\uff0cxb\u662f\u4e00\u4e2a4×8\u7684\u77e9\u9635\u3002\u7531\u4e8enn.Module<\/code> \u5b9e\u73b0\u4e86 __call__<\/code>\uff0c\u6240\u4ee5 m(...)<\/code> \u4f1a\u81ea\u52a8\u8f6c\u53bb\u8c03 forward<\/code>\u3002\u6240\u4ee5m(xb,yb)<\/code>\u5b9e\u9645\u4e0a\u662f\u8c03\u7528\u4e86forward<\/code>\u65b9\u6cd5\u3002<\/p>\n\n\n\n
\u60f3\u8c61\u4e00\u672c\u518c\u5b50\uff0c65 \u9875\uff0c\u6bcf\u9875\u5bf9\u5e94\u4e00\u4e2a\u5b57\u7b26\u3002\u7ffb\u5230 h<\/code> \u90a3\u4e00\u9875\uff0c\u4e0a\u9762\u5199\u7740 65 \u884c\u5c0f\u5b57\uff1a<\/p>\n\n\n\n
\u63a5 a \u7684\u53ef\u80fd\u6027\uff1a0.4\n\u63a5 b \u7684\u53ef\u80fd\u6027\uff1a-1.2\n...\n\u63a5 e \u7684\u53ef\u80fd\u6027\uff1a3.7   \u2190 \u6700\u9ad8\n...\n\u63a5 z \u7684\u53ef\u80fd\u6027\uff1a-0.8<\/pre>\n\n\n\n
\u8fd9\u4e00\u6574\u9875\uff0c\u5c31\u662f 65 \u4e2a\u6570\u3002<\/strong> \u4f60\u7ed9\u4e00\u4e2a\u5b57\u7b26\u7f16\u53f7\uff0c\u5b83\u628a\u90a3\u4e00\u9875\u539f\u6837\u6495\u7ed9\u4f60\u3002\u5c31\u8fd9\u4e48\u7b80\u5355\uff0c\u6ca1\u6709\u4efb\u4f55\u8ba1\u7b97\u3002\u8fd9\u5c31\u662fself.token_embedding_table(idx)<\/code> \u5e72\u7684\u4e8b\uff0c\u5c31\u662f \u62ff\u7740 idx<\/code> \u91cc\u7684\u6bcf\u4e2a\u7f16\u53f7\u53bb\u7ffb\u9875<\/strong>\u3002<\/p>\n\n\n\n
\u5173\u952e\u96be\u70b9\uff1aidx<\/code> \u4e0d\u662f\u4e00\u4e2a\u6570\uff0c\u662f\u4e00\u4e2a\u65b9\u5757<\/h4>\n\n\n\n
\u5982\u679c idx<\/code> \u53ea\u662f\u5355\u4e2a\u6574\u6570 46\uff0c\u90a3\u8fd4\u56de\u4e00\u9875 65 \u4e2a\u6570\uff0c\u597d\u7406\u89e3\u3002<\/p>\n\n\n\n
\u4f46 idx<\/code> \u662f xb<\/code>\uff0c\u5f62\u72b6 (4, 8)<\/code> \u2014\u2014 32 \u4e2a\u6574\u6570\u6392\u6210\u7684\u65b9\u5757\u3002\u4e8e\u662f Embedding \u8981\u7ffb 32 \u6b21\u9875\uff0c\u8fd4\u56de 32 \u9875\u3002<\/p>\n\n\n\n
\u8fd9 32 \u9875\u600e\u4e48\u6446\uff1f<\/strong> PyTorch \u7684\u89c4\u5219\u662f\uff1a\u539f\u6765\u6bcf\u4e2a\u6574\u6570\u5f85\u7684\u4f4d\u7f6e\uff0c\u73b0\u5728\u53d8\u6210\u4e00\u9875\u7eb8\u63d2\u5728\u90a3\u513f\u3002<\/strong><\/p>\n\n\n\n
\u6240\u4ee5out\u7684\u5f62\u72b6\u5c31\u53d8\u6210\u4e864x8x65\u4e86\u3002<\/p>\n\n\n\n
\u5b8c\u5584\u524d\u9879\u4f20\u64ad\u65b9\u6cd5<\/h3><\/div>\n\n\n\n
\u63a5\u7740\u6211\u4eec\u5b8c\u5584\u4e00\u4e0bforward<\/code>\u65b9\u6cd5\uff0c\u628a\u635f\u5931\u503c\u7b97\u51fa\u6765\uff1a<\/p>\n\n\n\n
class BigramLanguageModel(nn.Module):\n\n    def __init__(self, vocab_size):\n        super().__init__()\n        self.token_embedding_table = nn.Embedding(vocab_size, vocab_size)\n\n    def forward(self, idx, targets):\n\n        logits = self.token_embedding_table(idx)\n\n        B, T, C = logits.shape\n        logits = logits.view(B*T,C)\n        targets = targets.view(B*T)\n        loss = F.cross_entropy(logits, targets)\n        return logits, loss\n\nm = BigramLanguageModel(vocab_size)\nlogits, loss = m(xb,yb)\nprint(logits.shape)\nprint(loss)<\/code><\/pre>\n\n\n\n
\u8f93\u51fa\uff1a<\/p>\n\n\n\n
torch.Size([32, 65])\ntensor(5.0364, grad_fn=<NllLossBackward0>)<\/code><\/pre>\n\n\n\n
\u53ef\u4ee5\u770b\u5230\u521d\u59cb\u635f\u5931\u662f5.0364\u3002\u5176\u4e2d\u8fd9\u4e00\u6b65\uff1alogits = logits.view(B*T,C)\uff0c\u628a\u4e09\u7ef4\u62cd\u6241\u6210\u4e8c\u7ef4\uff0c\u56e0\u4e3aF.cross_entropy<\/code> \u6709\u4e25\u683c\u7684\u8f93\u5165\u683c\u5f0f\u8981\u6c42\uff1a<\/p>\n\n\n\n
    \n
  • \u9884\u6d4b\uff1a(N, C)<\/code> \u2014\u2014 N \u9053\u9898\uff0c\u6bcf\u9898 C \u4e2a\u9009\u9879\u7684\u5f97\u5206<\/li>\n\n\n\n
  • \u7b54\u6848\uff1a(N,)<\/code> \u2014\u2014 N \u4e2a\u6b63\u786e\u9009\u9879\u7684\u7f16\u53f7<\/li>\n<\/ul>\n\n\n\n
    \u800c\u6211\u4eec\u624b\u4e0a\u662f (4, 8, 65)<\/code> \u4e09\u7ef4\uff0c\u5b83\u4e0d\u6536\u3002<\/p>\n\n\n\n
    \u4f46\u4ed4\u7ec6\u60f3\u60f3\uff1a(4, 8)<\/code> \u8fd9\u4e24\u7ef4\u5176\u5b9e\u6ca1\u6709\u672c\u8d28\u533a\u522b\u30024 \u662f”\u54ea\u4e00\u6bb5”\uff0c8 \u662f”\u6bb5\u91cc\u7b2c\u51e0\u4e2a\u5b57\u7b26”\u2014\u2014\u5bf9 loss \u6765\u8bf4\uff0c\u8fd9 32 \u4e2a\u4f4d\u7f6e\u5c31\u662f 32 \u9053\u5f7c\u6b64\u72ec\u7acb\u7684\u9009\u62e9\u9898<\/strong>\uff0c\u6765\u81ea\u54ea\u4e00\u6bb5\u3001\u6392\u7b2c\u51e0\u4f4d\uff0c\u7edf\u7edf\u4e0d\u91cd\u8981\u3002<\/p>\n\n\n\n
    \u6240\u4ee5\u628a\u524d\u4e24\u7ef4\u63c9\u6210\u4e00\u7ef4\u5c31\u884c\uff1a<\/p>\n\n\n\n
    logits:  (4, 8, 65)  \u2192  (32, 65)\ntargets: (4, 8)      \u2192  (32,)<\/pre>\n\n\n\n
    view<\/code> \u53ea\u662f\u6362\u4e00\u79cd\u65b9\u5f0f\u770b\u540c\u4e00\u5757\u5185\u5b58\uff0c\u4e0d\u590d\u5236\u6570\u636e\uff0c\u4e0d\u6539\u53d8\u4efb\u4f55\u6570\u503c\uff0c\u7eaf\u7cb9\u662f\u91cd\u65b0\u89e3\u91ca\u5f62\u72b6\u3002\u6240\u4ee5\uff0c\u57fa\u672c\u4e0a\u4e5f\u6ca1\u6709\u4ec0\u4e48\u65f6\u95f4\u635f\u5931\u3002<\/p>\n\n\n\n
    \u5b9e\u73b0\u8f93\u51fa\u65b9\u6cd5<\/h3><\/div>\n\n\n\n
    \u63a5\u7740\u6211\u4eec\u5b9e\u73b0\u4e00\u4e0bgenerate<\/code>\u65b9\u6cd5\uff0c\u8ba9\u8fd9\u4e2a\u8fd8\u6ca1\u6709\u88ab\u8bad\u7ec3\u8fc7\u7684\u6a21\u578b\u80fd\u591f\u8f93\u51fa\uff1a<\/p>\n\n\n\n
    class BigramLanguageModel(nn.Module):\n\n    def __init__(self, vocab_size):\n        super().__init__()\n        self.token_embedding_table = nn.Embedding(vocab_size, vocab_size)\n\n    def forward(self, idx, targets=None):\n\n        logits = self.token_embedding_table(idx)\n\n        if targets is None:\n            loss = None\n        else:\n            B, T, C = logits.shape\n            logits = logits.view(B * T, C)\n            targets = targets.view(B * T)\n            loss = F.cross_entropy(logits, targets)\n\n        return logits, loss\n\n    def generate(self, idx, max_new_token):\n        for _ in range(max_new_token):\n            logits , loss = self.forward(idx)\n            logits = logits[:, -1, :]\n            probs = F.softmax(logits, dim=-1)\n            idx_next = torch.multinomial(probs, num_samples=1)\n            idx = torch.cat((idx, idx_next), dim = 1)\n        return idx\n\nm = BigramLanguageModel(vocab_size)\nlogits, loss = m(xb,yb)\nprint(logits.shape)\nprint(loss)\n\nidx = torch.zeros((1, 1), dtype=torch.long)\ncode = m.generate(idx, max_new_token=100)\nprint(decode(code[0].tolist()))<\/code><\/pre>\n\n\n\n
    \u8f93\u51fa\uff1a<\/p>\n\n\n\n
    lfJeukRuaRJKXAYtXzfJ:HEPiu--sDioi;ILCo3pHNTmDwJsfheKRxZCFs\nlZJ XQc?:s:HEzEnXalEPklcPU cL'DpdLCafBheH<\/code><\/pre>\n\n\n\n
    \u53ef\u4ee5\u770b\u5230\uff0c\u8fd9\u4e2a\u5b8c\u5168\u672a\u7ecf\u8bad\u7ec3\u7684\u6a21\u578b\u8f93\u51fa\u7684\u662f\u4e00\u56e2\u4e71\u7801\u3002generate<\/code>\u65b9\u6cd5\u7684\u6838\u5fc3\u601d\u8def\u5176\u5b9e\u5c31\u662f\uff1a\u731c\u4e00\u4e2a\u5b57 \u2192 \u63a5\u5230\u672b\u5c3e \u2192 \u62ff\u65b0\u5e8f\u5217\u518d\u731c\u4e0b\u4e00\u4e2a\uff0c\u5faa\u73af\u5f80\u590d\u3002\u8d77\u59cb\u662f\u4e00\u4e2a1×1\u7684\u5f20\u91cf\uff1aidx = torch.zeros((1, 1), dtype=torch.long)\uff0c\u4e5f\u5c31\u662f\u8d77\u59cb\u7b26\u3002<\/p>\n\n\n\n
    \u8bad\u7ec3\u6a21\u578b<\/h3><\/div>\n\n\n\n
    \u63a5\u4e0b\u6765\u6211\u4eec\u7528Adam\u65b9\u6cd5\u6765\u8bad\u7ec3\u6a21\u578b\uff0c\u5148\u628a\u5b66\u4e60\u7387\u8c03\u4e3a0.001\uff0c\u8bad\u7ec310000\u6b21\uff0c\u7136\u540e\u770b\u770b\u8f93\u51fa\uff0c\u4ee5\u4e0b\u662f\u5f53\u524d\u6240\u9700\u7684\u5b8c\u6574\u4ee3\u7801\uff1a<\/p>\n\n\n\n
    import torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\ntorch.manual_seed(1337)\n\nwith open('input.txt', 'r', encoding='utf-8') as f:\n    text = f.read()\nchars = sorted(list(set(text)))\nvocab_size = len(chars)\n\nstoi = { ch:i for i,ch in enumerate(chars) }\nitos = {i:ch for i,ch in enumerate(chars)}\nencode = lambda s: [stoi[c] for c in s]\ndecode = lambda l: ''.join([itos[i] for i in l])\n\n\n# \u5c06\u6570\u636e\u8f6c\u4e3a\u4e00\u7ef4\u5f20\u91cf\ndata = torch.tensor(encode(text), dtype=torch.long)\n\n# \u5c06\u6570\u636e\u5206\u4e3a\u8bad\u7ec3\u96c6\u548c\u9a8c\u8bc1\u96c6\nn = int(0.9*len(data))\ntrain_data = data[:n]\nval_data = data[n:]\n\nbatch_size = 4 # \u6279\u6b21\u5927\u5c0f\nblock_size = 8 # \u5757\u5927\u5c0f\ndef get_batch(split):\n    data = train_data if split == 'train' else val_data # \u533a\u5206\u52a0\u8f7d\u8bad\u7ec3\u96c6\u8fd8\u662f\u9a8c\u8bc1\u96c6\n    ix = torch.randint(len(data) - block_size,(batch_size,)) # \u968f\u673a\u8d77\u70b9\n    x = torch.stack([data[i:i+block_size] for i in ix])\n    y = torch.stack([data[i+1:i+block_size+1] for i in ix])\n    return x,y\n\nclass BigramLanguageModel(nn.Module):\n\n    def __init__(self, vocab_size):\n        super().__init__()\n        self.token_embedding_table = nn.Embedding(vocab_size, vocab_size)\n\n    def forward(self, idx, targets=None):\n\n        logits = self.token_embedding_table(idx)\n\n        if targets is None:\n            loss = None\n        else:\n            B, T, C = logits.shape\n            logits = logits.view(B * T, C)\n            targets = targets.view(B * T)\n            loss = F.cross_entropy(logits, targets)\n\n        return logits, loss\n\n    def generate(self, idx, max_new_token):\n        for _ in range(max_new_token):\n            logits , loss = self.forward(idx)\n            logits = logits[:, -1, :]\n            probs = F.softmax(logits, dim=-1)\n            idx_next = torch.multinomial(probs, num_samples=1)\n            idx = torch.cat((idx, idx_next), dim = 1)\n        return idx\n\nm = BigramLanguageModel(vocab_size)\n\noptimizer = torch.optim.Adam(m.parameters(), lr=1e-3)\n\nbatch_size = 32\nfor steps in range(10000):\n    xb, yb = get_batch('train')\n\n    logits ,loss = m(xb, yb)\n    optimizer.zero_grad(set_to_none=True)\n    loss.backward()\n    optimizer.step()\n\nxb, yb = get_batch('train')\nlogits ,loss = m(xb, yb)\nprint('loss: ', loss.item())\nidx = torch.zeros((1, 1), dtype=torch.long)\ncode = m.generate(idx, max_new_token=100)\nprint(decode(code[0].tolist()))<\/code><\/pre>\n\n\n\n
    \u8f93\u51fa\uff1a<\/p>\n\n\n\n
    loss:  2.46570086479187\n\nGLOMur.\nFRora?\nIEDUMBuselofor;\nTouthery the, mor, athe y ho!\nWhore aco tur! ms\nLI'd che, gat f hou w<\/code><\/pre>\n\n\n\n
    \u53ef\u4ee5\u770b\u5230\u635f\u5931\u503c\u964d\u5230\u4e862.46\uff0c\u8f93\u51fa\u7684\u4e5f\u6bd4\u4e4b\u524d\u8981\u597d\u4e86\uff0c\u4f46\u662f\u7531\u4e8e\u8fd9\u4e2a\u662f\u4e00\u4e2a\u4e8c\u5143\u6a21\u578b\uff0c\u6bcf\u6b21\u8f93\u51fa\u53ea\u770b\u524d\u9762\u7684\u4e00\u4e2a\u5b57\u7b26\uff0c\u5bfc\u81f4\u6548\u679c\u4e0d\u597d\uff0c\u6240\u4ee5\u6211\u4eec\u63a5\u4e0b\u6765\u9700\u8981\u8ba9\u5b57\u7b26\u4e4b\u95f4\u6709\u8054\u7cfb\u3002<\/p>\n\n\n\n
    \u63d0\u9ad8\u635f\u5931\u8ba1\u7b97\u51c6\u786e\u5ea6<\/h3><\/div>\n\n\n\n
    \u5355\u4e2a batch \u7684 loss \u566a\u58f0\u592a\u5927\uff0c\u4f1a\u5bfc\u81f4\u635f\u5931\u503c\u4e0d\u662f\u666e\u904d\u73b0\u8c61\uff0c\u6240\u4ee5\u6211\u4eec\u53ef\u4ee5\u8fde\u62bd 200 \u4e2a batch\uff0c\u53d6\u5e73\u5747\uff1a<\/p>\n\n\n\n
    @torch.no_grad()\ndef estimate_loss():\n    out = {}\n    model.eval()\n    for split in ['train', 'val']:\n        losses = torch.zeros(eval_iters)\n        for k in range(eval_iters):\n            X, Y = get_batch(split)\n            logits, loss = model(X, Y)\n            losses[k] = loss.item()\n        out[split] = losses.mean()\n    model.train()\n    return out<\/code><\/pre>\n\n\n\n
    \u53e6\u5916\uff0c\u6211\u4eec\u5bf9\u4ee3\u7801\u8fdb\u884c\u4e00\u4e9b\u4f18\u5316\uff0c\u5e76\u4e14\u5728\u53ef\u80fd\u7684\u60c5\u51b5\u4e0b\u4f7f\u7528GPU\u3002\u4ee5\u4e0b\u662f\u5f53\u524d\u7684\u5b8c\u6574\u4ee3\u7801\uff1a<\/p>\n\n\n\n
    import torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\ntorch.manual_seed(1337)\n\nbatch_size = 32 # \u6279\u6b21\u5927\u5c0f\nblock_size = 8 # \u5757\u5927\u5c0f\nmax_iters = 3000\neval_interval = 300\nlearning_rate = 1e-3\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\neval_iters = 200\nn_embd = 32\n\nwith open('input.txt', 'r', encoding='utf-8') as f:\n    text = f.read()\nchars = sorted(list(set(text)))\nvocab_size = len(chars)\n\nstoi = { ch:i for i,ch in enumerate(chars) }\nitos = {i:ch for i,ch in enumerate(chars)}\nencode = lambda s: [stoi[c] for c in s]\ndecode = lambda l: ''.join([itos[i] for i in l])\n\n\n# \u5c06\u6570\u636e\u8f6c\u4e3a\u4e00\u7ef4\u5f20\u91cf\ndata = torch.tensor(encode(text), dtype=torch.long)\n\n# \u5c06\u6570\u636e\u5206\u4e3a\u8bad\u7ec3\u96c6\u548c\u9a8c\u8bc1\u96c6\nn = int(0.9*len(data))\ntrain_data = data[:n]\nval_data = data[n:]\n\n\ndef get_batch(split):\n    data = train_data if split == 'train' else val_data # \u533a\u5206\u52a0\u8f7d\u8bad\u7ec3\u96c6\u8fd8\u662f\u9a8c\u8bc1\u96c6\n    ix = torch.randint(len(data) - block_size,(batch_size,)) # \u968f\u673a\u8d77\u70b9\n    x = torch.stack([data[i:i+block_size] for i in ix])\n    y = torch.stack([data[i+1:i+block_size+1] for i in ix])\n    x,y = x.to(device), y.to(device)\n    return x,y\n\n@torch.no_grad()\ndef estimate_loss():\n    out = {}\n    model.eval()\n    for split in ['train', 'val']:\n        losses = torch.zeros(eval_iters)\n        for k in range(eval_iters):\n            X, Y = get_batch(split)\n            logits, loss = model(X, Y)\n            losses[k] = loss.item()\n        out[split] = losses.mean()\n    model.train()\n    return out\n\nclass BigramLanguageModel(nn.Module):\n\n    def __init__(self):\n        super().__init__()\n        self.token_embedding_table = nn.Embedding(vocab_size, vocab_size)\n\n    def forward(self, idx, targets=None):\n\n        logits = self.token_embedding_table(idx)\n\n        if targets is None:\n            loss = None\n        else:\n            B, T, C = logits.shape\n            logits = logits.view(B * T, C)\n            targets = targets.view(B * T)\n            loss = F.cross_entropy(logits, targets)\n\n        return logits, loss\n\n    def generate(self, idx, max_new_token):\n        for _ in range(max_new_token):\n            logits , loss = self.forward(idx)\n            logits = logits[:, -1, :]\n            probs = F.softmax(logits, dim=-1)\n            idx_next = torch.multinomial(probs, num_samples=1)\n            idx = torch.cat((idx, idx_next), dim = 1)\n        return idx\n\nmodel = BigramLanguageModel()\nm = model.to(device)\n\noptimizer = torch.optim.Adam(m.parameters(), lr=learning_rate)\n\nfor steps in range(max_iters):\n    if steps % eval_interval == 0:\n        losses = estimate_loss()\n        print(f\"step {steps}: train loss {losses['train']:.4f}, val loss {losses['val']:.4f}\")\n    xb, yb = get_batch('train')\n\n    logits ,loss = m(xb, yb)\n    optimizer.zero_grad(set_to_none=True)\n    loss.backward()\n    optimizer.step()\n\nxb, yb = get_batch('train')\nlogits ,loss = m(xb, yb)\nprint('loss: ', loss.item())\nidx = torch.zeros((1, 1), dtype=torch.long, device=device)\ncode = m.generate(idx, max_new_token=100)\nprint(decode(code[0].tolist()))<\/code><\/pre>\n\n\n\n
    \u8dd1\u8d77\u6765\u770b\u770b\u8f93\u51fa\uff1a<\/p>\n\n\n\n
    step 0: train loss 4.7305, val loss 4.7241\nstep 300: train loss 4.3844, val loss 4.3922\nstep 600: train loss 4.0840, val loss 4.0824\nstep 900: train loss 3.8108, val loss 3.8160\nstep 1200: train loss 3.5885, val loss 3.5890\nstep 1500: train loss 3.3789, val loss 3.3861\nstep 1800: train loss 3.2206, val loss 3.2242\nstep 2100: train loss 3.0832, val loss 3.0824\nstep 2400: train loss 2.9669, val loss 2.9746\nstep 2700: train loss 2.8807, val loss 2.8798\nloss:  2.7016472816467285\n\n\n\nCExfikRO:\nwcowi,STAOLOL, btK\n\nHAPTombobeAUGe.\nSGJO-33SM:C?YIUauss:LVXEthafNusqhathe.t?ar dXlaSpate<\/code><\/pre>\n\n\n\n
    \u6570\u5b66\u94fa\u57ab<\/h3><\/div>\n\n\n\n
    \u63a5\u4e0b\u6765\uff0c\u8fd9\u91cc\u9700\u8981\u505a\u4e00\u70b9\u6570\u5b66\u94fa\u57ab\uff0c\u5173\u4e8e\u5982\u4f55\u8ba1\u7b97\u5e73\u5747\u503c\uff1a<\/p>\n\n\n\n
    \u5047\u8bbe\u00a0x<\/code>\u00a0\u662f 3 \u4e2a\u5b57\u7b26\uff0c\u6bcf\u4e2a\u5b57\u7b26\u7528\u4e8c\u7ef4\u5411\u91cf\u8868\u793a\uff1a<\/p>\n\n\n\n
    x = [[10, 20],\n     [30, 40],\n     [50, 60]]<\/code><\/pre>\n\n\n\n
    \u6211\u4eec\u60f3\u8981\u5f97\u5230\u6bcf\u4e2a\u5b57\u7b26\u548c\u5176\u524d\u9762\u7684\u5b57\u7b26\u7684\u5411\u91cf\u7684\u5e73\u5747\u503c\uff1a<\/p>\n\n\n\n
    xbow[0] = [10, 20]              \u5c31\u662f x[0]\nxbow[1] = [20, 30]              (x[0]+x[1])\/2\nxbow[2] = [30, 40]              (x[0]+x[1]+x[2])\/3<\/code><\/pre>\n\n\n\n
    \u6839\u636e\u77e9\u9635\u4e58\u6cd5\u7684\u89c4\u5219\uff0c\u6211\u4eec\u53ef\u4ee5\u8fd9\u6837\u5b9e\u73b0\uff1a<\/p>\n\n\n\n
    \"\"<\/div><\/figure><\/div>\n\n\n\n
    \u4f46\u662f\u8fd9\u4e2a\u5730\u65b9wei\u77e9\u9635\u5b83\u7684\u6743\u91cd\u90fd\u5199\u6b7b\u4e86\uff0c\u6211\u4eec\u5e0c\u671b\u5b83\u7684\u6743\u91cd\u7531\u6a21\u578b\u81ea\u5df1\u8fdb\u884c\u7b97\u51fa\u6765\uff0c\u6bcf\u4e2a\u4f4d\u7f6e\u81ea\u5df1\u51b3\u5b9a\u5173\u6ce8\u8c01\u3002\u6240\u4ee5\u6211\u4eec\u53ef\u4ee5\u628awei\u77e9\u9635\u7684\u53f3\u4e0a\u90e8\u5206\u90fd\u8bbe\u7f6e\u4e3a\u65e0\u7a77\u5c0f\uff0c\u4e4b\u540e\u8c03\u7528softmax\u65b9\u6cd5\uff0c\u5728softmax\u65b9\u6cd5\u4e2d\uff0c\u5148\u7684\u6bcf\u4e00\u4e2a\u4f4d\u7f6e\u6c42\u6307\u6570\uff0c\u90a3\u4e48\u4e3a0\u7684\u4f4d\u7f6e\u6c42\u6307\u6570\u4e4b\u540e\u5c31\u662f1\uff0c\u800c\u65e0\u7a77\u5c0f\u7684\u4f4d\u7f6e\u6c42\u6307\u6570\u4e4b\u540e\u5c31\u662f0\uff1a<\/p>\n\n\n\n
    \"\"<\/div><\/figure><\/div>\n\n\n\n
    \u589e\u52a0\u4f4d\u7f6e\u4fe1\u606f<\/h3><\/div>\n\n\n\n
    \u73b0\u5728\u8fd8\u662f\u53ea\u6709\u4e00\u4e2a65*65\u7684\u77e9\u9635\uff0c\u7528\u4e8e\u8868\u793a\u6bcf\u4e00\u4e2a\u5b57\u7b26\u4e4b\u540e\u6240\u6709\u5b57\u7b26\u7684\u53ef\u80fd\u6027\uff0c\u73b0\u5728\u6211\u4eec\u8981\u628a\u8fd9\u4e2a\u77e9\u9635\u62c6\u5f00\uff0c\u628aattention \u88c5\u5728\u8fd9\u91cc\uff1a<\/p>\n\n\n\n
    \"\"<\/div><\/figure><\/div>\n\n\n\n
    \u4e4b\u524d\u6bcf\u4e2a\u5b57\u7b26\u5bf9\u5e94\u7684\u662f65 \u4e2a\u5f97\u5206\uff0c\u73b0\u5728\u5bf9\u5e94\u7684\u662f32 \u4e2a\u6570\u3002\u8fd9 32 \u4e2a\u6570\u6ca1\u6709\u9884\u8bbe\u542b\u4e49\uff0c\u662f\u6a21\u578b\u81ea\u5df1\u5b66\u51fa\u6765\u7684\u7279\u5f81\u3002\u4f46\u662f\u8fd8\u7f3a\u4e00\u6837\u4e1c\u897f\uff1a\u4f4d\u7f6e\u4fe1\u606f\uff0c”abc” \u548c “cba” \u5e73\u5747\u51fa\u6765\u4e00\u6a21\u4e00\u6837\uff0c\u6240\u4ee5\u8981\u5355\u72ec\u5582\u4e00\u4efd\u4f4d\u7f6e\u4fe1\u606f\uff1a<\/p>\n\n\n\n
    \"\"<\/div><\/figure><\/div>\n\n\n\n
    \u8fd9\u662f\u6539\u9020\u540e\u7684BigramLanguageModel<\/code>\u7c7b\uff1a<\/p>\n\n\n\n
    class BigramLanguageModel(nn.Module):\n\n    def __init__(self):\n        super().__init__()\n        self.token_embedding_table = nn.Embedding(vocab_size, n_embd) # (65, 32)\n        self.position_embedding_table = nn.Embedding(block_size, n_embd) # (8, 32)\n        self.lm_head = nn.Linear(n_embd, vocab_size)\n\n    def forward(self, idx, targets=None):\n        B, T = idx.shape # B=4\u3001T=8\n        tok_emb = self.token_embedding_table(idx) # (4, 8, 32)\n        pos_emb = self.position_embedding_table(torch.arange(T, device=device)) # (8, 32)\n        x = tok_emb + pos_emb # (4, 8, 32) + (8, 32)  ->  (4, 8, 32) + (1, 8, 32) -> (4, 8, 32)\n        logits = self.lm_head(x) # (4, 8, 32) \u2192 (4, 8, 65)\n\n        if targets is None:\n            loss = None\n        else:\n            B, T, C = logits.shape\n            logits = logits.view(B * T, C)\n            targets = targets.view(B * T)\n            loss = F.cross_entropy(logits, targets)\n\n        return logits, loss\n\n    def generate(self, idx, max_new_token):\n        for _ in range(max_new_token):\n            idx_cond = idx[:, -block_size:]\n            logits , loss = self.forward(idx_cond)\n            logits = logits[:, -1, :]\n            probs = F.softmax(logits, dim=-1)\n            idx_next = torch.multinomial(probs, num_samples=1)\n            idx = torch.cat((idx, idx_next), dim = 1)\n        return idx<\/code><\/pre>\n\n\n\n
    \u589e\u52a0\u81ea\u6ce8\u610f\u529b\u5934\u4e0e\u591a\u5934\u6ce8\u610f\u529b<\/h3><\/div>\n\n\n\n
    \u6211\u4eec\u73b0\u5728\u6765\u5b9e\u73b0\u4e00\u4e2aHead\u7c7b\uff0c\u7528\u6765\u5b9e\u73b0\u81ea\u6ce8\u610f\u529b\u5934\uff1a<\/p>\n\n\n\n
    class Head(nn.Module):\n    def __init__(self, head_size):\n        super().__init__()\n        self.key = nn.Linear(n_embd, head_size, bias=False)\n        self.query = nn.Linear(n_embd, head_size, bias=False)\n        self.value = nn.Linear(n_embd, head_size, bias=False)\n        self.register_buffer('tril', torch.tril(torch.ones(block_size, block_size)))\n\n    def forward(self, x):\n        B, T, C = x.shape\n        k = self.key(x)\n        q = self.query(x)\n\n        wei = q @ k.transpose(-2, -1) * k.shape[-1]**-0.5\n        wei = wei.masked_fill(self.tril[:T, :T] == 0, float('-inf'))\n        wei = F.softmax(wei, dim=-1)\n\n        v = self.value(x)\n        out = wei @ v\n        return out<\/code><\/pre>\n\n\n\n
    \u6211\u4eec\u5c06\u81ea\u6ce8\u610f\u529b\u5934\u52a0\u5165\u5230BigramLanguageModel\u4e2d\uff0c\u4ee5\u4e0b\u662f\u5b8c\u6574\u4ee3\u7801\uff1a<\/p>\n\n\n\n
    import torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\ntorch.manual_seed(1337)\n\nbatch_size = 4 # \u6279\u6b21\u5927\u5c0f\nblock_size = 8 # \u5757\u5927\u5c0f\nmax_iters = 5000\neval_interval = 300\nlearning_rate = 1e-3\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\neval_iters = 200\nn_embd = 32\n\nwith open('input.txt', 'r', encoding='utf-8') as f:\n    text = f.read()\nchars = sorted(list(set(text)))\nvocab_size = len(chars)\n\nstoi = { ch:i for i,ch in enumerate(chars) }\nitos = {i:ch for i,ch in enumerate(chars)}\nencode = lambda s: [stoi[c] for c in s]\ndecode = lambda l: ''.join([itos[i] for i in l])\n\n\n# \u5c06\u6570\u636e\u8f6c\u4e3a\u4e00\u7ef4\u5f20\u91cf\ndata = torch.tensor(encode(text), dtype=torch.long)\n\n# \u5c06\u6570\u636e\u5206\u4e3a\u8bad\u7ec3\u96c6\u548c\u9a8c\u8bc1\u96c6\nn = int(0.9*len(data))\ntrain_data = data[:n]\nval_data = data[n:]\n\n\ndef get_batch(split):\n    data = train_data if split == 'train' else val_data # \u533a\u5206\u52a0\u8f7d\u8bad\u7ec3\u96c6\u8fd8\u662f\u9a8c\u8bc1\u96c6\n    ix = torch.randint(len(data) - block_size,(batch_size,)) # \u968f\u673a\u8d77\u70b9\n    x = torch.stack([data[i:i+block_size] for i in ix])\n    y = torch.stack([data[i+1:i+block_size+1] for i in ix])\n    x,y = x.to(device), y.to(device)\n    return x,y\n\n@torch.no_grad()\ndef estimate_loss():\n    out = {}\n    model.eval()\n    for split in ['train', 'val']:\n        losses = torch.zeros(eval_iters)\n        for k in range(eval_iters):\n            X, Y = get_batch(split)\n            logits, loss = model(X, Y)\n            losses[k] = loss.item()\n        out[split] = losses.mean()\n    model.train()\n    return out\n\nclass Head(nn.Module):\n    def __init__(self, head_size):\n        super().__init__()\n        self.key = nn.Linear(n_embd, head_size, bias=False)\n        self.query = nn.Linear(n_embd, head_size, bias=False)\n        self.value = nn.Linear(n_embd, head_size, bias=False)\n        self.register_buffer('tril', torch.tril(torch.ones(block_size, block_size)))\n\n    def forward(self, x):\n        B, T, C = x.shape\n        k = self.key(x)\n        q = self.query(x)\n\n        wei = q @ k.transpose(-2, -1) * k.shape[-1]**-0.5\n        wei = wei.masked_fill(self.tril[:T, :T] == 0, float('-inf'))\n        wei = F.softmax(wei, dim=-1)\n\n        v = self.value(x)\n        out = wei @ v\n        return out\n\nclass BigramLanguageModel(nn.Module):\n\n    def __init__(self):\n        super().__init__()\n        self.token_embedding_table = nn.Embedding(vocab_size, n_embd) # (65, 32)\n        self.position_embedding_table = nn.Embedding(block_size, n_embd) # (8, 32)\n        self.lm_head = nn.Linear(n_embd, vocab_size)\n        self.sa_head = Head(n_embd)\n\n    def forward(self, idx, targets=None):\n        B, T = idx.shape # B=4\u3001T=8\n        tok_emb = self.token_embedding_table(idx) # (4, 8, 32)\n        pos_emb = self.position_embedding_table(torch.arange(T, device=device)) # (8, 32)\n        x = tok_emb + pos_emb # (4, 8, 32) + (8, 32)  ->  (4, 8, 32) + (1, 8, 32) -> (4, 8, 32)\n        x = self.sa_head(x)\n        logits = self.lm_head(x) # (4, 8, 32) \u2192 (4, 8, 65)\n\n        if targets is None:\n            loss = None\n        else:\n            B, T, C = logits.shape\n            logits = logits.view(B * T, C)\n            targets = targets.view(B * T)\n            loss = F.cross_entropy(logits, targets)\n\n        return logits, loss\n\n    def generate(self, idx, max_new_token):\n        for _ in range(max_new_token):\n            idx_cond = idx[:, -block_size:]\n            logits , loss = self.forward(idx_cond)\n            logits = logits[:, -1, :]\n            probs = F.softmax(logits, dim=-1)\n            idx_next = torch.multinomial(probs, num_samples=1)\n            idx = torch.cat((idx, idx_next), dim = 1)\n        return idx\n\nmodel = BigramLanguageModel()\nm = model.to(device)\n\noptimizer = torch.optim.Adam(m.parameters(), lr=learning_rate)\n\nfor steps in range(max_iters):\n    if steps % eval_interval == 0:\n        losses = estimate_loss()\n        print(f\"step {steps}: train loss {losses['train']:.4f}, val loss {losses['val']:.4f}\")\n    xb, yb = get_batch('train')\n\n    logits ,loss = m(xb, yb)\n    optimizer.zero_grad(set_to_none=True)\n    loss.backward()\n    optimizer.step()\n\nxb, yb = get_batch('train')\nlogits ,loss = m(xb, yb)\nprint('loss: ', loss.item())\nidx = torch.zeros((1, 1), dtype=torch.long, device=device)\ncode = m.generate(idx, max_new_token=100)\nprint(decode(code[0].tolist()))<\/code><\/pre>\n\n\n\n
    \u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
    step 0: train loss 4.2468, val loss 4.2452\nstep 300: train loss 2.9372, val loss 2.9573\nstep 600: train loss 2.6529, val loss 2.6701\nstep 900: train loss 2.5592, val loss 2.5667\nstep 1200: train loss 2.5202, val loss 2.5194\nstep 1500: train loss 2.4848, val loss 2.4950\nstep 1800: train loss 2.4650, val loss 2.4743\nstep 2100: train loss 2.4496, val loss 2.4532\nstep 2400: train loss 2.4354, val loss 2.4550\nstep 2700: train loss 2.4209, val loss 2.4457\nstep 3000: train loss 2.4331, val loss 2.4332\nstep 3300: train loss 2.4236, val loss 2.4326\nstep 3600: train loss 2.4129, val loss 2.4277\nstep 3900: train loss 2.4096, val loss 2.4269\nstep 4200: train loss 2.4024, val loss 2.4369\nstep 4500: train loss 2.3964, val loss 2.4086\nstep 4800: train loss 2.3891, val loss 2.4090\nloss:  2.2492098808288574\n\nWhent aknt,\nTI'wilen is s, ble\n\nHiset bobe ton.\nSthr-'ts mealitanss:\nWant he uw hat vet?\nF dilas ate<\/code><\/pre>\n\n\n\n
    \u53ef\u4ee5\u770b\u5230\uff0c\u52a0\u5165\u4e86\u81ea\u6ce8\u610f\u529b\u5934\u4e4b\u540e\uff0c\u6211\u4eec\u7684\u635f\u5931\u503c\u7ec8\u4e8e\u964d\u5230\u4e862.4\u4ee5\u4e0b\uff0c\u8bf4\u660e\u6211\u4eec\u7684\u5de5\u4f5c\u662f\u6709\u6548\u679c\u7684\uff0c\u4f46\u662f\u6548\u679c\u8fd8\u4e0d\u662f\u5f88\u597d\uff0c\u6240\u4ee5\u6211\u4eec\u53ef\u4ee5\u5c06\u8fd9\u4e00\u5c42\u81ea\u6ce8\u610f\u529b\u5934\u62c6\u5206\u62104\u4e2a\u6ce8\u610f\u529b\u5934\uff0c\u6bcf\u4e00\u4e2a\u6ce8\u610f\u529b\u5934\u662f8\u7ef4\u7684\uff0c\u8fd9\u6837\u53ef\u4ee5\u4ece\u66f4\u591a\u7684\u7ef4\u5ea6\u5b66\u4e60\u7279\u5f81\uff0c\u53ef\u4ee5\u8fbe\u5230\u66f4\u597d\u7684\u6548\u679c\uff1a<\/p>\n\n\n\n
    class MultiHeadAttention(nn.Module):\n\n    def __init__(self, num_heads, head_size):\n        super().__init__()\n        self.head = nn.ModuleList([Head(head_size) for _ in range(num_heads)])\n\n    def forward(self, x):\n        return torch.cat([h(x) for h in self.head], dim=-1)\n<\/code><\/pre>\n\n\n\n
    \u4ee5\u4e0b\u662f\u5b8c\u6574\u4ee3\u7801\uff1a<\/p>\n\n\n\n
    import torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\ntorch.manual_seed(1337)\n\nbatch_size = 32 # \u6279\u6b21\u5927\u5c0f\nblock_size = 8 # \u5757\u5927\u5c0f\nmax_iters = 5000\neval_interval = 300\nlearning_rate = 1e-3\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\neval_iters = 200\nn_embd = 32\nnum_head = 4\nhead_size = n_embd \/\/ num_head\n\nwith open('input.txt', 'r', encoding='utf-8') as f:\n    text = f.read()\nchars = sorted(list(set(text)))\nvocab_size = len(chars)\n\nstoi = { ch:i for i,ch in enumerate(chars) }\nitos = {i:ch for i,ch in enumerate(chars)}\nencode = lambda s: [stoi[c] for c in s]\ndecode = lambda l: ''.join([itos[i] for i in l])\n\n\n# \u5c06\u6570\u636e\u8f6c\u4e3a\u4e00\u7ef4\u5f20\u91cf\ndata = torch.tensor(encode(text), dtype=torch.long)\n\n# \u5c06\u6570\u636e\u5206\u4e3a\u8bad\u7ec3\u96c6\u548c\u9a8c\u8bc1\u96c6\nn = int(0.9*len(data))\ntrain_data = data[:n]\nval_data = data[n:]\n\n\ndef get_batch(split):\n    data = train_data if split == 'train' else val_data # \u533a\u5206\u52a0\u8f7d\u8bad\u7ec3\u96c6\u8fd8\u662f\u9a8c\u8bc1\u96c6\n    ix = torch.randint(len(data) - block_size,(batch_size,)) # \u968f\u673a\u8d77\u70b9\n    x = torch.stack([data[i:i+block_size] for i in ix])\n    y = torch.stack([data[i+1:i+block_size+1] for i in ix])\n    x,y = x.to(device), y.to(device)\n    return x,y\n\n@torch.no_grad()\ndef estimate_loss():\n    out = {}\n    model.eval()\n    for split in ['train', 'val']:\n        losses = torch.zeros(eval_iters)\n        for k in range(eval_iters):\n            X, Y = get_batch(split)\n            logits, loss = model(X, Y)\n            losses[k] = loss.item()\n        out[split] = losses.mean()\n    model.train()\n    return out\n\nclass Head(nn.Module):\n    def __init__(self, head_size):\n        super().__init__()\n        self.key = nn.Linear(n_embd, head_size, bias=False)\n        self.query = nn.Linear(n_embd, head_size, bias=False)\n        self.value = nn.Linear(n_embd, head_size, bias=False)\n        self.register_buffer('tril', torch.tril(torch.ones(block_size, block_size)))\n\n    def forward(self, x):\n        B, T, C = x.shape\n        k = self.key(x)\n        q = self.query(x)\n\n        wei = q @ k.transpose(-2, -1) * k.shape[-1]**-0.5\n        wei = wei.masked_fill(self.tril[:T, :T] == 0, float('-inf'))\n        wei = F.softmax(wei, dim=-1)\n\n        v = self.value(x)\n        out = wei @ v\n        return out\n\nclass MultiHeadAttention(nn.Module):\n\n    def __init__(self, num_heads, head_size):\n        super().__init__()\n        self.head = nn.ModuleList([Head(head_size) for _ in range(num_heads)])\n\n    def forward(self, x):\n        return torch.cat([h(x) for h in self.head], dim=-1)\n\nclass BigramLanguageModel(nn.Module):\n\n    def __init__(self):\n        super().__init__()\n        self.token_embedding_table = nn.Embedding(vocab_size, n_embd) # (65, 32)\n        self.position_embedding_table = nn.Embedding(block_size, n_embd) # (8, 32)\n        self.lm_head = nn.Linear(n_embd, vocab_size)\n        self.sa_head = MultiHeadAttention(num_head, head_size)\n\n    def forward(self, idx, targets=None):\n        B, T = idx.shape # B=4\u3001T=8\n        tok_emb = self.token_embedding_table(idx) # (4, 8, 32)\n        pos_emb = self.position_embedding_table(torch.arange(T, device=device)) # (8, 32)\n        x = tok_emb + pos_emb # (4, 8, 32) + (8, 32)  ->  (4, 8, 32) + (1, 8, 32) -> (4, 8, 32)\n        x = self.sa_head(x)\n        logits = self.lm_head(x) # (4, 8, 32) \u2192 (4, 8, 65)\n\n        if targets is None:\n            loss = None\n        else:\n            B, T, C = logits.shape\n            logits = logits.view(B * T, C)\n            targets = targets.view(B * T)\n            loss = F.cross_entropy(logits, targets)\n\n        return logits, loss\n\n    def generate(self, idx, max_new_token):\n        for _ in range(max_new_token):\n            idx_cond = idx[:, -block_size:]\n            logits , loss = self.forward(idx_cond)\n            logits = logits[:, -1, :]\n            probs = F.softmax(logits, dim=-1)\n            idx_next = torch.multinomial(probs, num_samples=1)\n            idx = torch.cat((idx, idx_next), dim = 1)\n        return idx\n\nmodel = BigramLanguageModel()\nm = model.to(device)\n\noptimizer = torch.optim.Adam(m.parameters(), lr=learning_rate)\n\nfor steps in range(max_iters):\n    if steps % eval_interval == 0:\n        losses = estimate_loss()\n        print(f\"step {steps}: train loss {losses['train']:.4f}, val loss {losses['val']:.4f}\")\n    xb, yb = get_batch('train')\n\n    logits ,loss = m(xb, yb)\n    optimizer.zero_grad(set_to_none=True)\n    loss.backward()\n    optimizer.step()\n\nxb, yb = get_batch('train')\nlogits ,loss = m(xb, yb)\nprint('loss: ', loss.item())\nidx = torch.zeros((1, 1), dtype=torch.long, device=device)\ncode = m.generate(idx, max_new_token=100)\nprint(decode(code[0].tolist()))<\/code><\/pre>\n\n\n\n
    \u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
    step 0: train loss 4.2439, val loss 4.2423\nstep 300: train loss 2.8098, val loss 2.8238\nstep 600: train loss 2.5846, val loss 2.6018\nstep 900: train loss 2.4978, val loss 2.4958\nstep 1200: train loss 2.4529, val loss 2.4427\nstep 1500: train loss 2.4035, val loss 2.4074\nstep 1800: train loss 2.3731, val loss 2.3826\nstep 2100: train loss 2.3495, val loss 2.3508\nstep 2400: train loss 2.3237, val loss 2.3410\nstep 2700: train loss 2.3079, val loss 2.3311\nstep 3000: train loss 2.3054, val loss 2.3099\nstep 3300: train loss 2.2920, val loss 2.2993\nstep 3600: train loss 2.2737, val loss 2.2956\nstep 3900: train loss 2.2641, val loss 2.2892\nstep 4200: train loss 2.2528, val loss 2.2928\nstep 4500: train loss 2.2426, val loss 2.2681\nstep 4800: train loss 2.2305, val loss 2.2603\nloss:  2.123969554901123\n\nWhent ak tryocowd,\nThis thert madisen bobe toe.\nSagr-'t thealleands: rant he us crome.\nWar dilth ane<\/code><\/pre>\n\n\n\n
    \u867d\u7136\u6548\u679c\u8fd8\u662f\u4e0d\u5c3d\u5982\u4eba\u610f\uff0c\u4f46\u662f\u6211\u4eec\u7684\u635f\u5931\u503c\u786e\u5b9e\u662f\u5728\u4e0b\u964d\u7684\u3002<\/p>\n\n\n\n
    \u589e\u52a0\u524d\u9988\u7f51\u7edc<\/h3><\/div>\n\n\n\n
    \u73b0\u5728\u7684\u6a21\u578b\u91cc\uff0cattention \u635e\u56de\u4fe1\u606f\u540e\u7acb\u523b\u5c31\u9001\u53bb\u9884\u6d4b\u4e86\uff0c\u4e2d\u95f4\u6ca1\u6709\u4efb\u4f55\u52a0\u5de5\u73af\u8282\u3002\u800c\u5206\u5de5\u5e94\u8be5\u662f\uff1a<\/p>\n\n\n\n
    attention = \u8de8\u4f4d\u7f6e\u4ea4\u6d41\uff0c\u628a\u524d\u6587\u7684\u4fe1\u606f\u642c\u8fc7\u6765
    FFN = \u6bcf\u4e2a\u4f4d\u7f6e\u81ea\u5df1\u6d88\u5316\uff0c\u60f3\u60f3\u8fd9\u4e9b\u4fe1\u606f\u610f\u5473\u7740\u4ec0\u4e48<\/p>\n\n\n\n
    \u800c\u4e14 attention \u5168\u662f\u7ebf\u6027\u8fd0\u7b97\uff09\uff0c\u6ca1\u6709\u975e\u7ebf\u6027\u3002\u800cFFN \u7684 ReLU \u662f\u6574\u4e2a\u6a21\u578b\u91cc\u771f\u6b63\u7684\u975e\u7ebf\u6027\u6765\u6e90\u3002<\/p>\n\n\n\n
    class FeedForward(nn.Module):\n    def __init__(self, n_embd):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(n_embd, n_embd),\n            nn.ReLU(),\n        )\n        \n    def forward(self, x):\n        return self.net(x)<\/code><\/pre>\n\n\n\n
    \u5728BigramLanguageModel\u4e2d\u52a0\u5165FFN\u5c42\uff1a<\/p>\n\n\n\n
    class BigramLanguageModel(nn.Module):\n\n    def __init__(self):\n        super().__init__()\n        self.token_embedding_table = nn.Embedding(vocab_size, n_embd) # (65, 32)\n        self.position_embedding_table = nn.Embedding(block_size, n_embd) # (8, 32)\n        self.lm_head = nn.Linear(n_embd, vocab_size)\n        self.sa_head = MultiHeadAttention(num_head, head_size)\n        self.ffwd = FeedForward(n_embd)\n\n    def forward(self, idx, targets=None):\n        B, T = idx.shape # B=4\u3001T=8\n        tok_emb = self.token_embedding_table(idx) # (4, 8, 32)\n        pos_emb = self.position_embedding_table(torch.arange(T, device=device)) # (8, 32)\n        x = tok_emb + pos_emb # (4, 8, 32) + (8, 32)  ->  (4, 8, 32) + (1, 8, 32) -> (4, 8, 32)\n        x = self.sa_head(x)\n        x = self.ffwd(x)\n        logits = self.lm_head(x) # (4, 8, 32) \u2192 (4, 8, 65)\n\n        if targets is None:\n            loss = None\n        else:\n            B, T, C = logits.shape\n            logits = logits.view(B * T, C)\n            targets = targets.view(B * T)\n            loss = F.cross_entropy(logits, targets)\n\n        return logits, loss\n\n    def generate(self, idx, max_new_token):\n        for _ in range(max_new_token):\n            idx_cond = idx[:, -block_size:]\n            logits , loss = self.forward(idx_cond)\n            logits = logits[:, -1, :]\n            probs = F.softmax(logits, dim=-1)\n            idx_next = torch.multinomial(probs, num_samples=1)\n            idx = torch.cat((idx, idx_next), dim = 1)\n        return idx<\/code><\/pre>\n\n\n\n
    \u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
    step 0: train loss 4.2038, val loss 4.2009\nstep 300: train loss 2.7736, val loss 2.7907\nstep 600: train loss 2.5722, val loss 2.5836\nstep 900: train loss 2.4995, val loss 2.4857\nstep 1200: train loss 2.4415, val loss 2.4370\nstep 1500: train loss 2.3883, val loss 2.3961\nstep 1800: train loss 2.3583, val loss 2.3628\nstep 2100: train loss 2.3327, val loss 2.3350\nstep 2400: train loss 2.3102, val loss 2.3277\nstep 2700: train loss 2.2879, val loss 2.3076\nstep 3000: train loss 2.2890, val loss 2.2863\nstep 3300: train loss 2.2730, val loss 2.2742\nstep 3600: train loss 2.2486, val loss 2.2684\nstep 3900: train loss 2.2425, val loss 2.2667\nstep 4200: train loss 2.2275, val loss 2.2657\nstep 4500: train loss 2.2147, val loss 2.2376\nstep 4800: train loss 2.2085, val loss 2.2217\nloss:  2.2788593769073486\n\n\nYre bef bridcowilen is sow the airet bube don.\nShirnd my dagatauss:\nWhith fou qor he.\nWar dils\nHate<\/code><\/pre>\n\n\n\n
    \u5b9e\u73b0\u591a\u5c42\u5806\u53e0<\/h3><\/div>\n\n\n\n
    \u6211\u4eec\u53ef\u4ee5\u5c06\u81ea\u6ce8\u610f\u529b\u5c42\u548c\u524d\u9988\u7f51\u7edc\u7ec4\u6210\u4e00\u4e2aBlock\uff0c\u518d\u8fdb\u884c\u591a\u5c42\u5806\u53e0\uff0c\u6765\u5b9e\u73b0\u6df1\u5ea6\u795e\u7ecf\u7f51\u7edc\uff1a<\/p>\n\n\n\n
    class Block(nn.Module):\n    def __init__(self, n_embd, n_head):\n        super().__init__()\n        self.sa = MultiHeadAttention(n_head, n_embd \/\/ n_head)\n        self.ffwd = FeedForward(n_embd)\n\n    def forward(self, x):\n        x = self.sa(x)\n        x = self.ffwd(x)\n        return x<\/code><\/pre>\n\n\n\n
    \u6b8b\u5dee\u8fde\u63a5<\/h3><\/div>\n\n\n\n
    \u4e3a\u4e86\u89e3\u51b3\u591a\u5c42\u5806\u53e0\u7684\u6df1\u5ea6\u795e\u7ecf\u7f51\u7edc\u968f\u7740\u5c42\u6570\u53d8\u591a\uff0c\u6548\u679c\u53cd\u800c\u53d8\u5dee\u7684\u95ee\u9898\uff0c\u6211\u4eec\u53ef\u4ee5\u901a\u8fc7\u6b8b\u5dee\u8fde\u63a5\u7684\u65b9\u5f0f\uff0c\u8ba9\u68af\u5ea6\u53ef\u4ee5\u8fdb\u884c\u6709\u6548\u7684\u4f20\u5bfc\uff1a<\/p>\n\n\n\n
    class Block(nn.Module):\n    def __init__(self, n_embd, n_head):\n        super().__init__()\n        self.sa = MultiHeadAttention(n_head, n_embd \/\/ n_head)\n        self.ffwd = FeedForward(n_embd)\n\n    def forward(self, x):\n        x = x + self.sa(x)\n        x = x + self.ffwd(x)\n        return x<\/code><\/pre>\n\n\n\n
    \u7ed9\u591a\u5934\u6ce8\u610f\u529b\u52a0\u4e00\u4e2a\u6295\u5f71\u5c42<\/h3><\/div>\n\n\n\n
    \u5728\u591a\u5934\u6ce8\u610f\u529b\u5c42\u540e\u9762\u52a0\u4e00\u4e2a\u6295\u5f71\u5c42\uff0c\u53ef\u4ee5\u5c06\u539f\u672c4\u4e2a\u591a\u5934\u6ce8\u610f\u529b\u5f97\u5230\u7684\u4fe1\u606f\u6c47\u603b\u6210\u4e00\u4e2a\uff0c\u8ba9\u6bcf\u4e00\u4e2a\u4f4d\u7f6e\u90fd\u80fd\u4eab\u53d7\u5230\u6bcf\u4e00\u5934\u6ce8\u610f\u529b\u7684\u8bad\u7ec3\u6548\u679c\uff1a<\/p>\n\n\n\n
    class MultiHeadAttention(nn.Module):\n\n    def __init__(self, num_heads, head_size):\n        super().__init__()\n        self.head = nn.ModuleList([Head(head_size) for _ in range(num_heads)])\n        self.proj = nn.Linear(n_embd, n_embd)\n\n    def forward(self, x):\n        out = torch.cat([h(x) for h in self.head], dim=-1)\n        out = self.proj(out)\n        return out<\/code><\/pre>\n\n\n\n
    \u628a FeedForward \u6539\u6210\u5bbd\u7248<\/h3><\/div>\n\n\n\n
    \u5728\u66f4\u5bbd\u7684\u7a7a\u95f4\u91cc\uff0cReLU \u80fd\u505a\u51fa\u66f4\u7cbe\u7ec6\u7684\u5224\u65ad\uff1a<\/p>\n\n\n\n
    class FeedForward(nn.Module):\n    def __init__(self, n_embd):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(n_embd, 4 * n_embd),\n            nn.ReLU(),\n            nn.Linear(4 * n_embd, n_embd),\n        )\n\n    def forward(self, x):\n        return self.net(x)<\/code><\/pre>\n\n\n\n
    \u7ed9 Block \u52a0\u5f52\u4e00\u5316<\/h3><\/div>\n\n\n\n
    \u7ed9Block\u52a0\u5165\u5f52\u4e00\u5316LayerNorm \u53ef\u4ee5\u89e3\u51b3\u6570\u503c\u6f02\u79fb\uff1a<\/p>\n\n\n\n
    class Block(nn.Module):\n    def __init__(self, n_embd, n_head):\n        super().__init__()\n        self.sa = MultiHeadAttention(n_head, n_embd \/\/ n_head)\n        self.ffwd = FeedForward(n_embd)\n        self.ln1 = nn.LayerNorm(n_embd)\n        self.ln2 = nn.LayerNorm(n_embd)\n\n    def forward(self, x):\n        x = x + self.sa(self.ln1(x))\n        x = x + self.ffwd(self.ln2(x))\n        return x<\/code><\/pre>\n\n\n\n
    \u4fee\u6539\u6a21\u578b<\/h3><\/div>\n\n\n\n
    \u73b0\u5728\u6211\u4eec\u5f00\u59cb\u7ec4\u88c5\u6a21\u578b\uff0c\u628aBlock\u52a0\u5165\u8fdb\u53bb\uff0c\u4ee5\u4e0b\u662f\u5b8c\u6574\u4ee3\u7801\uff1a<\/p>\n\n\n\n
    import torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\ntorch.manual_seed(1337)\n\nbatch_size = 32 # \u6279\u6b21\u5927\u5c0f\nblock_size = 8 # \u5757\u5927\u5c0f\nmax_iters = 5000\neval_interval = 300\nlearning_rate = 1e-3\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\neval_iters = 200\nn_embd = 32\nnum_head = 4\nhead_size = n_embd \/\/ num_head\nn_layer = 3\n\nwith open('input.txt', 'r', encoding='utf-8') as f:\n    text = f.read()\nchars = sorted(list(set(text)))\nvocab_size = len(chars)\n\nstoi = { ch:i for i,ch in enumerate(chars) }\nitos = {i:ch for i,ch in enumerate(chars)}\nencode = lambda s: [stoi[c] for c in s]\ndecode = lambda l: ''.join([itos[i] for i in l])\n\n\n# \u5c06\u6570\u636e\u8f6c\u4e3a\u4e00\u7ef4\u5f20\u91cf\ndata = torch.tensor(encode(text), dtype=torch.long)\n\n# \u5c06\u6570\u636e\u5206\u4e3a\u8bad\u7ec3\u96c6\u548c\u9a8c\u8bc1\u96c6\nn = int(0.9*len(data))\ntrain_data = data[:n]\nval_data = data[n:]\n\n\ndef get_batch(split):\n    data = train_data if split == 'train' else val_data # \u533a\u5206\u52a0\u8f7d\u8bad\u7ec3\u96c6\u8fd8\u662f\u9a8c\u8bc1\u96c6\n    ix = torch.randint(len(data) - block_size,(batch_size,)) # \u968f\u673a\u8d77\u70b9\n    x = torch.stack([data[i:i+block_size] for i in ix])\n    y = torch.stack([data[i+1:i+block_size+1] for i in ix])\n    x,y = x.to(device), y.to(device)\n    return x,y\n\n@torch.no_grad()\ndef estimate_loss():\n    out = {}\n    model.eval()\n    for split in ['train', 'val']:\n        losses = torch.zeros(eval_iters)\n        for k in range(eval_iters):\n            X, Y = get_batch(split)\n            logits, loss = model(X, Y)\n            losses[k] = loss.item()\n        out[split] = losses.mean()\n    model.train()\n    return out\n\nclass Head(nn.Module):\n    def __init__(self, head_size):\n        super().__init__()\n        self.key = nn.Linear(n_embd, head_size, bias=False)\n        self.query = nn.Linear(n_embd, head_size, bias=False)\n        self.value = nn.Linear(n_embd, head_size, bias=False)\n        self.register_buffer('tril', torch.tril(torch.ones(block_size, block_size)))\n\n    def forward(self, x):\n        B, T, C = x.shape\n        k = self.key(x)\n        q = self.query(x)\n\n        wei = q @ k.transpose(-2, -1) * k.shape[-1]**-0.5\n        wei = wei.masked_fill(self.tril[:T, :T] == 0, float('-inf'))\n        wei = F.softmax(wei, dim=-1)\n\n        v = self.value(x)\n        out = wei @ v\n        return out\n\nclass MultiHeadAttention(nn.Module):\n\n    def __init__(self, num_heads, head_size):\n        super().__init__()\n        self.head = nn.ModuleList([Head(head_size) for _ in range(num_heads)])\n        self.proj = nn.Linear(n_embd, n_embd)\n\n    def forward(self, x):\n        out = torch.cat([h(x) for h in self.head], dim=-1)\n        out = self.proj(out)\n        return out\n\nclass FeedForward(nn.Module):\n    def __init__(self, n_embd):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(n_embd, 4 * n_embd),\n            nn.ReLU(),\n            nn.Linear(4 * n_embd, n_embd),\n        )\n\n    def forward(self, x):\n        return self.net(x)\n\nclass Block(nn.Module):\n    def __init__(self, n_embd, n_head):\n        super().__init__()\n        self.sa = MultiHeadAttention(n_head, n_embd \/\/ n_head)\n        self.ffwd = FeedForward(n_embd)\n        self.ln1 = nn.LayerNorm(n_embd)\n        self.ln2 = nn.LayerNorm(n_embd)\n\n    def forward(self, x):\n        x = x + self.sa(self.ln1(x))\n        x = x + self.ffwd(self.ln2(x))\n        return x\n\nclass BigramLanguageModel(nn.Module):\n\n    def __init__(self):\n        super().__init__()\n        self.token_embedding_table = nn.Embedding(vocab_size, n_embd) # (65, 32)\n        self.position_embedding_table = nn.Embedding(block_size, n_embd) # (8, 32)\n        self.lm_head = nn.Linear(n_embd, vocab_size)\n        self.blocks = nn.Sequential(*[Block(n_embd, num_head) for _ in range(n_layer)])\n        self.ln_f = nn.LayerNorm(n_embd)\n\n    def forward(self, idx, targets=None):\n        B, T = idx.shape # B=4\u3001T=8\n        tok_emb = self.token_embedding_table(idx) # (4, 8, 32)\n        pos_emb = self.position_embedding_table(torch.arange(T, device=device)) # (8, 32)\n        x = tok_emb + pos_emb # (4, 8, 32) + (8, 32)  ->  (4, 8, 32) + (1, 8, 32) -> (4, 8, 32)\n        x = self.blocks(x)\n        x = self.ln_f(x)\n        logits = self.lm_head(x) # (4, 8, 32) \u2192 (4, 8, 65)\n\n        if targets is None:\n            loss = None\n        else:\n            B, T, C = logits.shape\n            logits = logits.view(B * T, C)\n            targets = targets.view(B * T)\n            loss = F.cross_entropy(logits, targets)\n\n        return logits, loss\n\n    def generate(self, idx, max_new_token):\n        for _ in range(max_new_token):\n            idx_cond = idx[:, -block_size:]\n            logits , loss = self.forward(idx_cond)\n            logits = logits[:, -1, :]\n            probs = F.softmax(logits, dim=-1)\n            idx_next = torch.multinomial(probs, num_samples=1)\n            idx = torch.cat((idx, idx_next), dim = 1)\n        return idx\n\nmodel = BigramLanguageModel()\nm = model.to(device)\n\noptimizer = torch.optim.Adam(m.parameters(), lr=learning_rate)\n\nfor steps in range(max_iters):\n    if steps % eval_interval == 0:\n        losses = estimate_loss()\n        print(f\"step {steps}: train loss {losses['train']:.4f}, val loss {losses['val']:.4f}\")\n    xb, yb = get_batch('train')\n\n    logits ,loss = m(xb, yb)\n    optimizer.zero_grad(set_to_none=True)\n    loss.backward()\n    optimizer.step()\n\nxb, yb = get_batch('train')\nlogits ,loss = m(xb, yb)\nprint('loss: ', loss.item())\nidx = torch.zeros((1, 1), dtype=torch.long, device=device)\ncode = m.generate(idx, max_new_token=100)\nprint(decode(code[0].tolist()))<\/code><\/pre>\n\n\n\n
    \u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
    step 0: train loss 4.3026, val loss 4.2987\nstep 300: train loss 2.5105, val loss 2.5183\nstep 600: train loss 2.3595, val loss 2.3622\nstep 900: train loss 2.2617, val loss 2.2782\nstep 1200: train loss 2.1920, val loss 2.2222\nstep 1500: train loss 2.1568, val loss 2.1908\nstep 1800: train loss 2.1267, val loss 2.1737\nstep 2100: train loss 2.0928, val loss 2.1419\nstep 2400: train loss 2.0855, val loss 2.1328\nstep 2700: train loss 2.0757, val loss 2.1268\nstep 3000: train loss 2.0499, val loss 2.0906\nstep 3300: train loss 2.0264, val loss 2.0967\nstep 3600: train loss 2.0195, val loss 2.0893\nstep 3900: train loss 2.0213, val loss 2.0897\nstep 4200: train loss 2.0038, val loss 2.0701\nstep 4500: train loss 1.9887, val loss 2.0669\nstep 4800: train loss 1.9666, val loss 2.0668\nloss:  1.8026065826416016\n\n\nYCORK:\nSery cowe,\nThe lay be madishombear toe.\nStirn' my day, and barththile were, to bar dilthoate\n<\/code><\/pre>\n\n\n\n
    \u5806\u53e0\u6210\u529f\u4e86\uff0c\u6709\u660e\u663e\u6548\u679c\uff01<\/p>\n\n\n\n
    \u6dfb\u52a0 Dropout<\/h3><\/div>\n\n\n\n
    Dropout\u4f1a\u5c06\u4e00\u90e8\u5206\u795e\u7ecf\u5143\u5173\u95ed\uff0c\u6211\u4eec\u63a5\u4e0b\u6765\u8981\u6781\u5927\u5730\u6269\u5c55\u6a21\u578b\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0cDropout\u5bf9\u9632\u6b62\u8fc7\u62df\u5408\u7684\u53d1\u751f\u6709\u4e00\u5b9a\u5e2e\u52a9\uff1a<\/p>\n\n\n\n
    import torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\ntorch.manual_seed(1337)\n\nbatch_size = 32 # \u6279\u6b21\u5927\u5c0f\nblock_size = 8 # \u5757\u5927\u5c0f\nmax_iters = 5000\neval_interval = 300\nlearning_rate = 1e-3\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\neval_iters = 200\nn_embd = 32\nnum_head = 4\nhead_size = n_embd \/\/ num_head\nn_layer = 3\ndropout = 0.2\n\nwith open('input.txt', 'r', encoding='utf-8') as f:\n    text = f.read()\nchars = sorted(list(set(text)))\nvocab_size = len(chars)\n\nstoi = { ch:i for i,ch in enumerate(chars) }\nitos = {i:ch for i,ch in enumerate(chars)}\nencode = lambda s: [stoi[c] for c in s]\ndecode = lambda l: ''.join([itos[i] for i in l])\n\n\n# \u5c06\u6570\u636e\u8f6c\u4e3a\u4e00\u7ef4\u5f20\u91cf\ndata = torch.tensor(encode(text), dtype=torch.long)\n\n# \u5c06\u6570\u636e\u5206\u4e3a\u8bad\u7ec3\u96c6\u548c\u9a8c\u8bc1\u96c6\nn = int(0.9*len(data))\ntrain_data = data[:n]\nval_data = data[n:]\n\n\ndef get_batch(split):\n    data = train_data if split == 'train' else val_data # \u533a\u5206\u52a0\u8f7d\u8bad\u7ec3\u96c6\u8fd8\u662f\u9a8c\u8bc1\u96c6\n    ix = torch.randint(len(data) - block_size,(batch_size,)) # \u968f\u673a\u8d77\u70b9\n    x = torch.stack([data[i:i+block_size] for i in ix])\n    y = torch.stack([data[i+1:i+block_size+1] for i in ix])\n    x,y = x.to(device), y.to(device)\n    return x,y\n\n@torch.no_grad()\ndef estimate_loss():\n    out = {}\n    model.eval()\n    for split in ['train', 'val']:\n        losses = torch.zeros(eval_iters)\n        for k in range(eval_iters):\n            X, Y = get_batch(split)\n            logits, loss = model(X, Y)\n            losses[k] = loss.item()\n        out[split] = losses.mean()\n    model.train()\n    return out\n\nclass Head(nn.Module):\n    def __init__(self, head_size):\n        super().__init__()\n        self.key = nn.Linear(n_embd, head_size, bias=False)\n        self.query = nn.Linear(n_embd, head_size, bias=False)\n        self.value = nn.Linear(n_embd, head_size, bias=False)\n        self.register_buffer('tril', torch.tril(torch.ones(block_size, block_size)))\n        self.dropout = nn.Dropout(dropout)\n\n    def forward(self, x):\n        B, T, C = x.shape\n        k = self.key(x)\n        q = self.query(x)\n\n        wei = q @ k.transpose(-2, -1) * k.shape[-1]**-0.5\n        wei = wei.masked_fill(self.tril[:T, :T] == 0, float('-inf'))\n        wei = F.softmax(wei, dim=-1)\n        wei = self.dropout(wei)\n\n        v = self.value(x)\n        out = wei @ v\n        return out\n\nclass MultiHeadAttention(nn.Module):\n\n    def __init__(self, num_heads, head_size):\n        super().__init__()\n        self.head = nn.ModuleList([Head(head_size) for _ in range(num_heads)])\n        self.proj = nn.Linear(n_embd, n_embd)\n        self.dropout = nn.Dropout(dropout)\n\n    def forward(self, x):\n        out = torch.cat([h(x) for h in self.head], dim=-1)\n        out = self.dropout(self.proj(out))\n        return out\n\nclass FeedForward(nn.Module):\n    def __init__(self, n_embd):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(n_embd, 4 * n_embd),\n            nn.ReLU(),\n            nn.Linear(4 * n_embd, n_embd),\n            nn.Dropout(dropout),\n        )\n\n    def forward(self, x):\n        return self.net(x)\n\nclass Block(nn.Module):\n    def __init__(self, n_embd, n_head):\n        super().__init__()\n        self.sa = MultiHeadAttention(n_head, n_embd \/\/ n_head)\n        self.ffwd = FeedForward(n_embd)\n        self.ln1 = nn.LayerNorm(n_embd)\n        self.ln2 = nn.LayerNorm(n_embd)\n\n    def forward(self, x):\n        x = x + self.sa(self.ln1(x))\n        x = x + self.ffwd(self.ln2(x))\n        return x\n\nclass BigramLanguageModel(nn.Module):\n\n    def __init__(self):\n        super().__init__()\n        self.token_embedding_table = nn.Embedding(vocab_size, n_embd) # (65, 32)\n        self.position_embedding_table = nn.Embedding(block_size, n_embd) # (8, 32)\n        self.lm_head = nn.Linear(n_embd, vocab_size)\n        self.blocks = nn.Sequential(*[Block(n_embd, num_head) for _ in range(n_layer)])\n        self.ln_f = nn.LayerNorm(n_embd)\n\n    def forward(self, idx, targets=None):\n        B, T = idx.shape # B=4\u3001T=8\n        tok_emb = self.token_embedding_table(idx) # (4, 8, 32)\n        pos_emb = self.position_embedding_table(torch.arange(T, device=device)) # (8, 32)\n        x = tok_emb + pos_emb # (4, 8, 32) + (8, 32)  ->  (4, 8, 32) + (1, 8, 32) -> (4, 8, 32)\n        x = self.blocks(x)\n        x = self.ln_f(x)\n        logits = self.lm_head(x) # (4, 8, 32) \u2192 (4, 8, 65)\n\n        if targets is None:\n            loss = None\n        else:\n            B, T, C = logits.shape\n            logits = logits.view(B * T, C)\n            targets = targets.view(B * T)\n            loss = F.cross_entropy(logits, targets)\n\n        return logits, loss\n\n    def generate(self, idx, max_new_token):\n        for _ in range(max_new_token):\n            idx_cond = idx[:, -block_size:]\n            logits , loss = self.forward(idx_cond)\n            logits = logits[:, -1, :]\n            probs = F.softmax(logits, dim=-1)\n            idx_next = torch.multinomial(probs, num_samples=1)\n            idx = torch.cat((idx, idx_next), dim = 1)\n        return idx\n\nmodel = BigramLanguageModel()\nm = model.to(device)\n\noptimizer = torch.optim.Adam(m.parameters(), lr=learning_rate)\n\nfor steps in range(max_iters):\n    if steps % eval_interval == 0:\n        losses = estimate_loss()\n        print(f\"step {steps}: train loss {losses['train']:.4f}, val loss {losses['val']:.4f}\")\n    xb, yb = get_batch('train')\n\n    logits ,loss = m(xb, yb)\n    optimizer.zero_grad(set_to_none=True)\n    loss.backward()\n    optimizer.step()\n\nxb, yb = get_batch('train')\nlogits ,loss = m(xb, yb)\nprint('loss: ', loss.item())\nidx = torch.zeros((1, 1), dtype=torch.long, device=device)\ncode = m.generate(idx, max_new_token=100)\nprint(decode(code[0].tolist()))<\/code><\/pre>\n\n\n\n
    \u6269\u5927\u6a21\u578b<\/h3><\/div>\n\n\n\n
    \u73b0\u5728\u6211\u4eec\u5c06\u6a21\u578b\u8fdb\u884c\u5927\u89c4\u6a21\u7684\u653e\u5927\uff0c\u4fee\u6539\u914d\u7f6e\u533a\uff1a<\/p>\n\n\n\n
    batch_size = 64 # \u6279\u6b21\u5927\u5c0f\nblock_size = 256 # \u5757\u5927\u5c0f\nmax_iters = 5000\neval_interval = 500\nlearning_rate = 3e-4\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\neval_iters = 200\nn_embd = 384\nnum_head = 6\nhead_size = n_embd \/\/ num_head\nn_layer = 6\ndropout = 0.2<\/code><\/pre>\n\n\n\n
    \u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
    step 0: train loss 4.2835, val loss 4.2872\nstep 500: train loss 1.8878, val loss 1.9991\nstep 1000: train loss 1.5391, val loss 1.7281\nstep 1500: train loss 1.3933, val loss 1.6026\nstep 2000: train loss 1.3073, val loss 1.5525\nstep 2500: train loss 1.2483, val loss 1.5242\nstep 3000: train loss 1.2022, val loss 1.5059\nstep 3500: train loss 1.1609, val loss 1.4910\nstep 4000: train loss 1.1213, val loss 1.4901\nstep 4500: train loss 1.0867, val loss 1.4871\nloss:  1.1334030628204346\n\nKING EDWARD IV:\nWhat, grant, lord, will you just now,\nThus you repose the true of the king,\nThore no<\/code><\/pre>\n\n\n\n
    \u53ef\u4ee5\u770b\u5230\uff0c\u635f\u5931\u5df2\u7ecf\u964d\u52301.08\u4e86\uff0c\u9a8c\u8bc1\u96c6\u7684\u635f\u5931\u4e5f\u964d\u5230\u4e861.4\uff0c\u8bf4\u660e\u6211\u4eec\u7684\u5de5\u4f5c\u662f\u975e\u5e38\u5bcc\u6709\u6210\u6548\u7684\u3002<\/p>\n\n\n\n
    \u7ed3\u8bed<\/h2>\n\n\n\n
    \u8fd9\u6b21\uff0c\u6211\u4eec\u5b8c\u6574\u5730\u5b9e\u73b0\u4e86\u4e00\u6b21Transformer\u67b6\u6784\u7684\u795e\u7ecf\u7f51\u7edc\uff0c\u5e76\u4e14\u8ba9\u5176\u8f93\u51fa\u4e86\u838e\u58eb\u6bd4\u4e9a\u98ce\u683c\u7684\u6587\u672c\u3002\u4ece\u4e00\u5f00\u59cb\u7684\u4e8c\u5143\u8bed\u8a00\u6a21\u578b\u5230\u7ed3\u5c3e\u7684Transformer\u67b6\u6784\u6a21\u578b\uff0c\u6211\u4eec\u5b66\u5230\u4e86\u5f88\u591a\u3002\u4e5f\u5bf9\u6211\u4eec\u5b66\u4e60\u6df1\u5ea6\u795e\u7ecf\u7f51\u7edc\u6709\u6781\u5927\u7684\u5e2e\u52a9\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"
    \u8fd9\u662fAndrej Karpathy \u96f6\u57fa\u7840\u5b66\u4e60\u795e\u7ecf\u7f51\u7edc\u7684\u89c6\u9891\u8bfe\u7684\u7b2c\u4e03\u8282\u8bfe\u3002\u672c\u6b21\u6211\u4eec\u5c06\u4f1a\u5b66\u4e60\u5982\u4f55\u642d\u5efa\u4e00\u4e2aTran […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_uag_custom_page_level_css":"","footnotes":""},"categories":[10],"tags":[29,24,28],"class_list":["post-2476","post","type-post","status-publish","format-standard","hentry","category-10","tag-andrej-karpathy","tag-24","tag-28"],"uagb_featured_image_src":{"full":false,"thumbnail":false,"medium":false,"medium_large":false,"large":false,"1536x1536":false,"2048x2048":false},"uagb_author_info":{"display_name":"zjml","author_link":"https:\/\/blog.zjml.life\/?author=1"},"uagb_comment_info":0,"uagb_excerpt":"\u8fd9\u662fAndrej Karpathy \u96f6\u57fa\u7840\u5b66\u4e60\u795e\u7ecf\u7f51\u7edc\u7684\u89c6\u9891\u8bfe\u7684\u7b2c\u4e03\u8282\u8bfe\u3002\u672c\u6b21\u6211\u4eec\u5c06\u4f1a\u5b66\u4e60\u5982\u4f55\u642d\u5efa\u4e00\u4e2aTran…","_links":{"self":[{"href":"https:\/\/blog.zjml.life\/index.php?rest_route=\/wp\/v2\/posts\/2476","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/blog.zjml.life\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/blog.zjml.life\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/blog.zjml.life\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/blog.zjml.life\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=2476"}],"version-history":[{"count":3,"href":"https:\/\/blog.zjml.life\/index.php?rest_route=\/wp\/v2\/posts\/2476\/revisions"}],"predecessor-version":[{"id":2480,"href":"https:\/\/blog.zjml.life\/index.php?rest_route=\/wp\/v2\/posts\/2476\/revisions\/2480"}],"wp:attachment":[{"href":"https:\/\/blog.zjml.life\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=2476"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/blog.zjml.life\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=2476"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/blog.zjml.life\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=2476"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}