{"id":2425,"date":"2026-08-13T17:04:54","date_gmt":"2026-08-13T09:04:54","guid":{"rendered":"https:\/\/blog.zjml.life\/?p=2425"},"modified":"2026-08-13T17:07:45","modified_gmt":"2026-08-13T09:07:45","slug":"%e8%b7%9f%e7%9d%80-andrej-karpathy-%e9%9b%b6%e5%9f%ba%e7%a1%80%e5%ad%a6%e4%b9%a0%e7%a5%9e%e7%bb%8f%e7%bd%91%e7%bb%9c-%e5%a4%9a%e5%b1%82%e6%84%9f%e7%9f%a5%e6%9c%ba-mlp-%e5%ad%a6%e4%b9%a0%e7%ac%94","status":"publish","type":"post","link":"https:\/\/blog.zjml.life\/?p=2425","title":{"rendered":"\u8ddf\u7740 Andrej Karpathy \u96f6\u57fa\u7840\u5b66\u4e60\u795e\u7ecf\u7f51\u7edc \u591a\u5c42\u611f\u77e5\u673a (MLP) \u5b66\u4e60\u7b14\u8bb0"},"content":{"rendered":"\n

\u8fd9\u662fAndrej Karpathy \u96f6\u57fa\u7840\u5b66\u4e60\u795e\u7ecf\u7f51\u7edc\u7684\u89c6\u9891\u8bfe\u7684\u7b2c\u4e09\u8282\u8bfe\uff0c\u5728\u8fd9\u4e00\u8282\u8bfe\u4e2d\uff0c\u6211\u4eec\u4f1a\u5b9e\u73b0\u4e00\u4e2a\u7b80\u5355\u7684\u82f1\u6587\u540d\u751f\u6210\u7f51\u7edc\u3002<\/p>\n\n\n\n

\u51c6\u5907\u5de5\u4f5c<\/h2>\n\n\n\n

\u5b89\u88c5Python
\u5b89\u88c5Pytorch
\u89c2\u770b\u89c6\u9891\u8bfe\u7a0b<\/a><\/p>\n\n\n\n

\u6b63\u5f0f\u5f00\u59cb\u7f16\u5199\u4ee3\u7801<\/h2>\n\n\n\n

\u51c6\u5907\u59d3\u540d\u96c6\u548c\u5b57\u5178<\/h3>\n\n\n\n

\u9996\u5148\uff0c\u6211\u4eec\u9700\u8981\u51c6\u5907\u597d\u59d3\u540d\u6587\u672c\uff0c\u8fd9\u53ef\u4ee5\u4eceGitHub\u4e0a\u76f4\u63a5\u4e0b\u8f7d<\/a>\uff0c\u8bfb\u8005\u53ef\u4ee5\u70b9\u51fb\u76f4\u63a5\u4e0b\u8f7d<\/a>\u8fdb\u884c\u4e0b\u8f7d\u3002\u7b14\u8005\u5df2\u7ecf\u628a\u94fe\u63a5\u8bbe\u7f6e\u597d\u4e86\u3002<\/p>\n\n\n\n

\u6211\u4eec\u5148\u6309\u7167\u4e0a\u8282\u8bfe\u7684\u601d\u8def\uff0c\u8bfb\u53d6\u59d3\u540d\u96c6\uff0c\u5e76\u4e14\u521b\u5efa\u5b57\u7b26\u6620\u5c04\u6574\u6570\u7d22\u5f15\u7684\u5b57\u5178\u548c\u53cd\u5411\u5b57\u5178\uff1a<\/p>\n\n\n\n

import torch\nimport torch.nn.functional as F\nimport matplotlib.pyplot as plt\n\nwords = open(\"names.txt\",\"r\").read().splitlines()\n\nchars = sorted(list(set(\"\".join(words))))\nstoi = { s : i+1 for i,s in enumerate(chars) } # \u5b57\u7b26\u6620\u5c04\u6574\u6570\u7d22\u5f15\nstoi[\".\"] = 0\nitos = { i : s for s,i in stoi.items() } # \u53cd\u5411\u5b57\u5178\nprint(stoi)\nprint(itos)<\/code><\/pre>\n\n\n\n
\u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
{'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5, 'f': 6, 'g': 7, 'h': 8, 'i': 9, 'j': 10, 'k': 11, 'l': 12, 'm': 13, 'n': 14, 'o': 15, 'p': 16, 'q': 17, 'r': 18, 's': 19, 't': 20, 'u': 21, 'v': 22, 'w': 23, 'x': 24, 'y': 25, 'z': 26, '.': 0}\n\n{1: 'a', 2: 'b', 3: 'c', 4: 'd', 5: 'e', 6: 'f', 7: 'g', 8: 'h', 9: 'i', 10: 'j', 11: 'k', 12: 'l', 13: 'm', 14: 'n', 15: 'o', 16: 'p', 17: 'q', 18: 'r', 19: 's', 20: 't', 21: 'u', 22: 'v', 23: 'w', 24: 'x', 25: 'y', 26: 'z', 0: '.'}<\/code><\/pre>\n\n\n\n
\u51c6\u5907\u8bad\u7ec3\u6837\u672c<\/h3>\n\n\n\n
\u63a5\u4e0b\u6765\u6211\u4eec\u9700\u8981\u51c6\u5907\u8bad\u7ec3\u7528\u7684\u6837\u672c\uff0c\u4e5f\u5c31\u662f\u8f93\u5165\u4ec0\u4e48\u3001\u8f93\u51fa\u4ec0\u4e48\u3002\u6211\u4eec\u7684\u8f93\u5165\u53ef\u4ee5\u6539\u4e3a\u524d\u9762\u4e09\u4e2a\u8fde\u7eed\u5b57\u7b26\uff0c\u800c\u4e0d\u662f\u4e0a\u4e00\u8282\u8bfe\u7684\u524d\u4e00\u4e2a\u5b57\u7b26\uff0c\u8f93\u51fa\u5c31\u662f\u7b2c\u56db\u4e2a\u5b57\u7b26\uff0c\u5f53\u7136\u8fd9\u91cc\u7684\u5b57\u7b26\u6211\u4eec\u8981\u7528\u521a\u521a\u51c6\u5907\u597d\u7684\u5b57\u5178\uff0c\u6620\u5c04\u4e3a\u6570\u5b57\u3002\u6211\u4eec\u91cd\u65b0\u5199\u4e00\u4e2amain.py\uff1a<\/p>\n\n\n\n
import torch\nimport torch.nn.functional as F\nimport matplotlib.pyplot as plt\n\nwords = open(\"names.txt\",\"r\").read().splitlines()\n\n# \u521b\u5efa\u5b57\u5178\u548c\u53cd\u5411\u5b57\u5178\nchars = sorted(list(set(\"\".join(words))))\nstoi = { s : i+1 for i,s in enumerate(chars) } # \u5b57\u7b26\u6620\u5c04\u6574\u6570\u7d22\u5f15\nstoi[\".\"] = 0\nitos = { i : s for s,i in stoi.items() } # \u53cd\u5411\u5b57\u5178\n\nblock_size = 3\nX, Y = [], []\nfor word in words[:1]:\n    # \u5148\u5728\u524d\u9762\u52a0\u4e0ablock_size\u4e2a\u201c.\u201d\uff0c\u4e5f\u5c31\u662f\u52a0\u4e0a\u82e5\u5e72\u4e2a0\uff0c\u56e0\u4e3a\u662f\u6309\u7167stoi\u4e2d\u7684\u6570\u5b57\u6765\u4fdd\u5b58\u7684\n    context = [0] * block_size\n    for char in word + \".\":\n        ix = stoi[char]\n        X.append(context)\n        Y.append(ix)\n        # \u5c06context\u5f80\u540e\u632a\u4e00\u4e2a\u5b57\u7b26\n        context = context[1:] + [ix]\n        print(X)\n        print(Y)<\/code><\/pre>\n\n\n\n
\u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
[[0, 0, 0]]\n[5]\n[[0, 0, 0], [0, 0, 5]]\n[5, 13]\n[[0, 0, 0], [0, 0, 5], [0, 5, 13]]\n[5, 13, 13]\n[[0, 0, 0], [0, 0, 5], [0, 5, 13], [5, 13, 13]]\n[5, 13, 13, 1]\n[[0, 0, 0], [0, 0, 5], [0, 5, 13], [5, 13, 13], [13, 13, 1]]\n[5, 13, 13, 1, 0]<\/code><\/pre>\n\n\n\n
\u5b9e\u73b0\u5d4c\u5165\u5c42<\/h3>\n\n\n\n
\u8fd9\u662f\u975e\u5e38\u5173\u952e\u7684\u4e00\u6b65\uff0c\u7531\u4e8e\u6211\u4eec\u73b0\u5728\u662f\u901a\u8fc7\u524d\u9762\u4e09\u4e2a\u5b57\u7b26\u6765\u63a8\u5bfc\u540e\u9762\u4e00\u4e2a\u5b57\u7b26\uff0c\u5982\u679c\u6211\u4eec\u8fd8\u50cf\u4e0a\u8282\u8bfe\u90a3\u6837\u4f7f\u7528one_hot\u7684\u65b9\u5f0f\u5411\u4e0b\u4e00\u5c42\u4f20\u9012\u6837\u672c\u7684\u8f93\u5165\uff0c\u90a3\u4e48\u8fd9\u4e00\u5c42\u5c31\u9700\u898127\u00d73=81\u4e2a\u8f93\u5165\uff0c\u5e76\u4e14\u5982\u679c\u672a\u6765\u6211\u4eec\u9700\u8981\u6839\u636e\u524d5\u4e2a\u3001\u524d10\u4e2a\u5b57\u7b26\u6765\u63a8\u5bfc\u4e0b\u4e00\u4e2a\u5b57\u7b26\u7684\u8bdd\uff0c\u8fd9\u91cc\u4f1a\u53d8\u5f97\u66f4\u5927\uff0c\u56e0\u6b64\u6211\u4eec\u901a\u8fc7\u4e00\u4e2a\u5d4c\u5165\u5c42\u5c06\u6bcf\u4e00\u4e2a\u5b57\u7b26\u8f6c\u5316\u4e3a\u4e24\u4e2a\u6d6e\u70b9\u6570\u6765\u8868\u793a\u3002\u5728\u8fd9\u91cc\u6211\u4eec\u60f3\u8ba9\u6bcf\u4e00\u4e2a\u5b57\u7b26\u90fd\u6709\u4e24\u4e2a\u6d6e\u70b9\u6570\u6765\u8868\u793a\uff0c\u4e5f\u5c31\u662f\u4e00\u4e2a\u5d4c\u5165\u5c42\u3002
\u56e0\u4e3a\u6bcf\u4e00\u4e2a\u5b57\u7b26\u90fd\u6709\u4e24\u4e2a\u6d6e\u70b9\u6570\u6765\u8868\u793a\uff0c\u6240\u4ee5\u5b83\u53ef\u4ee5\u8868\u8fbe\u5b57\u7b26\u8ddf\u5b57\u7b26\u4e4b\u95f4\u7684\u4f4d\u7f6e\u5173\u7cfb\u3002\u6bd4\u5982\u5982\u679c\u6709\u4e24\u4e2a\u5b57\u7b26\u5728\u82f1\u6587\u4e2d\u6709\u5f88\u591a\u7684\u5171\u901a\u70b9\uff0c\u90a3\u4e48\u5b83\u4eec\u7684\u8ddd\u79bb\u5c31\u4f1a\u53d8\u5f97\u5f88\u8fd1\uff0c\u53cd\u4e4b\u5219\u4f1a\u53d8\u5f97\u5f88\u8fdc\u3002\u5b9e\u9645\u9879\u76ee\u4e2d\uff0c\u6211\u4eec\u901a\u5e38\u4f1a\u4f7f\u752850\u4e2a\u751a\u81f3100\u4e2a\u6d6e\u70b9\u6570\u6765\u8868\u793a\u4e00\u4e2a\u5b57\u7b26\u6216\u8005\u4e00\u4e2a\u8bcd\u8bed\u3002\u5728\u8fd9\u91cc\u6211\u4eec\u4e3a\u4e86\u65b9\u4fbf\u6f14\u793a\uff0c\u9009\u62e9\u75282\u4e2a\u6d6e\u70b9\u6570\u7684\u8868\u793a\u4e00\u4e2a\u5b57\u7b26\u3002
\u867d\u7136\u4e00\u5f00\u59cb\u6211\u4eec\u662f\u968f\u673a\u7684\u4e00\u4e2a27\u00d72\u7684\u53c2\u6570\u8868\uff0c\u4f46\u662f\u5728\u672a\u6765\u901a\u8fc7\u53cd\u5411\u4f20\u64ad\uff0c\u6211\u4eec\u4f1a\u9010\u6e10\u5730\u8c03\u6574\u8fd9\u4e2a\u53c2\u6570\u8868\u66f4\u52a0\u51c6\u786e\u3002<\/p>\n\n\n\n
C = torch.randn((27,2)) # \u5d4c\u5165\u5c42\nemb = C[torch.tensor(X)] # \u5c06\u6837\u672c\u4e2d\u7684\u5b57\u7b26\u4f9d\u6b21\u8f6c\u5316\u4e3a\u6d6e\u70b9\u6570\u8868\u793a\nprint(emb)<\/code><\/pre>\n\n\n\n
\u6211\u4eec\u770b\u4e00\u4e0b\u8f93\u51fa\uff1a<\/p>\n\n\n\n
tensor([[[ 1.0239, -0.8605],\n         [ 1.0239, -0.8605],\n         [ 1.0239, -0.8605]],\n\n        [[ 1.0239, -0.8605],\n         [ 1.0239, -0.8605],\n         [ 0.4716,  0.6099]],\n\n        [[ 1.0239, -0.8605],\n         [ 0.4716,  0.6099],\n         [ 0.1226,  0.3713]],\n\n        [[ 0.4716,  0.6099],\n         [ 0.1226,  0.3713],\n         [ 0.1226,  0.3713]],\n\n        [[ 0.1226,  0.3713],\n         [ 0.1226,  0.3713],\n         [ 0.2849, -1.1253]]])<\/code><\/pre>\n\n\n\n
\u5b9e\u73b0\u9690\u85cf\u5c42\u548c\u8f93\u51fa\u5c42<\/h3>\n\n\n\n
\u53ef\u4ee5\u770b\u5230\uff0c\u6837\u672c\u4e2d\u7684\u6bcf\u4e00\u4e2a\u5b57\u7b26\u90fd\u8f6c\u6362\u4e3a\u4e86\u4e00\u4e2a\u7531\u4e24\u4e2a\u6d6e\u70b9\u6570\u8868\u793a\u7684\u5217\u8868\u3002\u4e0b\u4e00\u6b65\uff0c\u5c06\u6bcf\u4e00\u4e2a\u6837\u672c\u4e2d\u7684\u8fd96\u4e2a\u6d6e\u70b9\u6570\u6539\u4e3a\u4e00\u4e2a\u5217\u8868\uff0c\u8f93\u51fa\u7ed9\u4e0b\u4e00\u5c42\u9690\u85cf\u5c42\u3002\u8fd96\u4e2a\u6d6e\u70b9\u6570\u6839\u636e\u987a\u5e8f\u4f9d\u6b21\u6392\u5217\uff0c\u8fd9\u6837\u5c31\u53ef\u4ee5\u4f53\u73b0\u51fa\u6837\u672c\u4e2d\u5b57\u7b26\u7684\u5148\u540e\u987a\u5e8f\u3002\u901a\u8fc7\u6211\u4eec\u7684\u8fd9\u4e00\u6b65\u64cd\u4f5c\uff0c\u539f\u5148\u9700\u8981\u5c06\u9690\u85cf\u5c42\u4f20\u8f9381\u4e2a\u8f93\u5165\uff0c\u800c\u73b0\u5728\u6211\u4eec\u53ea\u9700\u8981\u4f20\u51656\u4e2a\u6d6e\u70b9\u6570\u4e86\uff0c\u5927\u5927\u7684\u51cf\u5c11\u4e86\u4f20\u5165\u7684\u91cf\u3002<\/p>\n\n\n\n
W1 = torch.randn((6,100))\nb1 = torch.randn(100)\n\nh = emb.view(-1,6) @ W1+b1\nprint(h.shape)<\/code><\/pre>\n\n\n\n
emb.view(-1,6)<\/code>\u8fd9\u91cc\u7684-1\u7528\u4e8e\u8868\u793a\u8ba9PyTorch\u81ea\u884c\u63a8\u5bfc\u8fd9\u91cc\u5e94\u8be5\u662f\u591a\u5c11\u3002\u8fd9\u91cc\u5b9e\u9645\u4e0a\u662f\u6837\u672c\u7684\u6570\u91cf\uff0c\u6bd4\u5982\u5728\u73b0\u5728\u5b83\u4f1a\u63a8\u5bfc\u51fa\u6765\u662f5\uff0c\u56e0\u4e3a\u6211\u4eec\u5728\u521a\u521a\u53ea\u4f7f\u7528\u4e86\u59d3\u540d\u6837\u672c\u96c6\u4e2d\u7684\u6700\u540e\u4e00\u4e2a\u59d3\u540d\uff0c\u7684\u6837\u672c\u6570\u91cf\u53ea\u67095\u3002\u5982\u679c\u6211\u4eec\u7684\u6837\u672c\u6709\u4e0a\u4e07\uff0c\u90a3\u8fd9\u4e2a\u5730\u65b9\u4f1a\u63a8\u5bfc\u51fa\u4e0a\u4e07\u3002<\/p>\n\n\n\n
\u6211\u4eec\u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
torch.Size([5, 100])<\/code><\/pre>\n\n\n\n
\u53ef\u4ee5\u770b\u5230\uff0c\u6211\u4eec\u8fd9\u4e00\u5c42\u7ed9\u4e0b\u4e00\u5c42\u795e\u7ecf\u7f51\u7edc\u63d0\u4f9b\u4e86100\u4e2a\u8f93\u51fa\uff0c\u90a3\u4e48\u4e0b\u4e00\u4e2a\u9690\u85cf\u5c42\u5c31\u8981\u63d0\u4f9b100\u4e2a\u8f93\u5165\u3002\u7531\u4e8e\u6211\u4eec\u5728\u8fd9\u91cc\u53ea\u5e0c\u671b\u5b9e\u73b0\u4e24\u5c42\u795e\u7ecf\u7f51\u7edc\uff0c\u56e0\u6b64\uff0c\u4e0b\u4e00\u5c42\u795e\u7ecf\u7f51\u7edc\u8981\u670927\u4e2a\u8f93\u51fa\uff0c\u5bf9\u5e9427\u4e2a\u5b57\u7b26\uff1a<\/p>\n\n\n\n
W2 = torch.randn((100,27))\nb2 = torch.randn(27)\nlogits = h @ W2 + b2\nprint(logits.shape)<\/code><\/pre>\n\n\n\n
\u6211\u4eec\u6765\u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
torch.Size([5, 27])<\/code><\/pre>\n\n\n\n
\u53ef\u4ee5\u770b\u5230\uff0c\u6211\u4eec\u8f93\u51fa\u5c42\u5df2\u7ecf\u5b9e\u73b0\u4e8627\u4e2a\u8f93\u51fa\u3002<\/p>\n\n\n\n
\u56fa\u5b9a\u968f\u673a\u79cd\u5b50<\/h3>\n\n\n\n
\u4e3a\u4e86\u63a5\u4e0b\u6765\u66f4\u597d\u7684\u6f14\u793a\u548c\u590d\u73b0\uff0c\u6211\u4fee\u6539\u4e86\u4ee3\u7801\uff0c\u56fa\u5b9a\u4e86\u968f\u673a\u79cd\u5b50\u3002\u4ee5\u4e0b\u662f\u5f53\u524d\u7684\u5b8c\u6574\u4ee3\u7801\uff1a<\/p>\n\n\n\n
import torch\nimport torch.nn.functional as F\nimport matplotlib.pyplot as plt\n\nwords = open(\"names.txt\",\"r\").read().splitlines()\ng = torch.Generator().manual_seed(2147483647) # \u56fa\u5b9a\u968f\u673a\u79cd\u5b50\n\n# \u521b\u5efa\u5b57\u5178\u548c\u53cd\u5411\u5b57\u5178\nchars = sorted(list(set(\"\".join(words))))\nstoi = { s : i+1 for i,s in enumerate(chars) } # \u5b57\u7b26\u6620\u5c04\u6574\u6570\u7d22\u5f15\nstoi[\".\"] = 0\nitos = { i : s for s,i in stoi.items() } # \u53cd\u5411\u5b57\u5178\n\nblock_size = 3\nX, Y = [], []\nfor word in words[:1]:\n    # \u5148\u5728\u524d\u9762\u52a0\u4e0ablock_size\u4e2a\u201c.\u201d\uff0c\u4e5f\u5c31\u662f\u52a0\u4e0a\u82e5\u5e72\u4e2a0\uff0c\u56e0\u4e3a\u662f\u6309\u7167stoi\u4e2d\u7684\u6570\u5b57\u6765\u4fdd\u5b58\u7684\n    context = [0] * block_size\n    for char in word + \".\":\n        ix = stoi[char]\n        X.append(context)\n        Y.append(ix)\n        # \u5c06context\u5f80\u540e\u632a\u4e00\u4e2a\u5b57\u7b26\n        context = context[1:] + [ix]\n\nC = torch.randn((27,2), generator=g) # \u5d4c\u5165\u5c42\nemb = C[torch.tensor(X)] # \u5c06\u6837\u672c\u4e2d\u7684\u5b57\u7b26\u4f9d\u6b21\u8f6c\u5316\u4e3a\u6d6e\u70b9\u6570\u8868\u793a\n\n# \u7b2c\u4e00\u5c42\u9690\u85cf\u5c42\nW1 = torch.randn((6,100), generator=g)\nb1 = torch.randn(100, generator=g)\nh = emb.view(-1,6) @ W1+b1\n\n# \u7b2c\u4e8c\u5c42\u8f93\u51fa\u5c42\nW2 = torch.randn((100,27), generator=g)\nb2 = torch.randn(27, generator=g)\nlogits = h @ W2 + b2<\/code><\/pre>\n\n\n\n
\u8f93\u51fa\u5f52\u4e00\u5316\u4e0e\u8ba1\u7b97\u635f\u5931<\/h3>\n\n\n\n
\u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u518d\u5bf9\u8f93\u51fa\u5c42\u7684\u8f93\u51fa\u8fdb\u884c\u5f52\u4e00\u5316\uff0c\u5e76\u4e14\u8ba1\u7b97\u635f\u5931\uff1a<\/p>\n\n\n\n
counts = logits.exp()
prob = counts \/ counts.sum(1, keepdims=True)
loss = -prob[torch.arange(5),Y].log().mean()<\/pre>\n\n\n\n
\u5f53\u7136\uff0c\u8ba1\u7b97loss\u53ef\u4ee5\u4f7f\u7528Pytorch\u7684\u65b9\u6cd5\uff0c\u6211\u4eec\u65e0\u9700\u91cd\u590d\u9020\u8f6e\u5b50\uff1a<\/p>\n\n\n\n
loss = F.cross_entropy(logits, torch.tensor(Y))\nprint(loss)<\/code><\/pre>\n\n\n\n
\u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
tensor(99.6126)<\/code><\/pre>\n\n\n\n
\u6a21\u578b\u8bad\u7ec3<\/h3>\n\n\n\n
\u63a5\u4e0b\u6765\u5c31\u662f\u6b63\u5411\u4f20\u64ad\u4e0e\u53cd\u5411\u4f20\u64ad\uff0c\u6211\u4eec\u4fee\u6539\u4ee3\u7801\uff0c\u8ba9\u5176\u80fd\u591f\u8fdb\u884c\u5fae\u8c03\u7684\u8bad\u7ec3\uff1a<\/p>\n\n\n\n
import torch\nimport torch.nn.functional as F\nimport matplotlib.pyplot as plt\n\nwords = open(\"names.txt\",\"r\").read().splitlines()\ng = torch.Generator().manual_seed(2147483647) # \u56fa\u5b9a\u968f\u673a\u79cd\u5b50\n\n# \u521b\u5efa\u5b57\u5178\u548c\u53cd\u5411\u5b57\u5178\nchars = sorted(list(set(\"\".join(words))))\nstoi = { s : i+1 for i,s in enumerate(chars) } # \u5b57\u7b26\u6620\u5c04\u6574\u6570\u7d22\u5f15\nstoi[\".\"] = 0\nitos = { i : s for s,i in stoi.items() } # \u53cd\u5411\u5b57\u5178\n\nblock_size = 3\nX, Y = [], []\nfor word in words[:5]:\n    # \u5148\u5728\u524d\u9762\u52a0\u4e0ablock_size\u4e2a\u201c.\u201d\uff0c\u4e5f\u5c31\u662f\u52a0\u4e0a\u82e5\u5e72\u4e2a0\uff0c\u56e0\u4e3a\u662f\u6309\u7167stoi\u4e2d\u7684\u6570\u5b57\u6765\u4fdd\u5b58\u7684\n    context = [0] * block_size\n    for char in word + \".\":\n        ix = stoi[char]\n        X.append(context)\n        Y.append(ix)\n        # \u5c06context\u5f80\u540e\u632a\u4e00\u4e2a\u5b57\u7b26\n        context = context[1:] + [ix]\n\nC = torch.randn((27,2), generator=g) # \u5d4c\u5165\u5c42\n\n# \u7b2c\u4e00\u5c42\u9690\u85cf\u5c42\nW1 = torch.randn((6,100), generator=g)\nb1 = torch.randn(100, generator=g)\n\n# \u7b2c\u4e8c\u5c42\u8f93\u51fa\u5c42\nW2 = torch.randn((100,27), generator=g)\nb2 = torch.randn(27, generator=g)\n\nparameters = [C,W1,W2,b1,b2]\nfor p in parameters:\n    p.requires_grad = True\n\nfor _ in range(100):\n    emb = C[torch.tensor(X)]  # \u5c06\u6837\u672c\u4e2d\u7684\u5b57\u7b26\u4f9d\u6b21\u8f6c\u5316\u4e3a\u6d6e\u70b9\u6570\u8868\u793a\n    h = emb.view(-1, 6) @ W1 + b1\n    logits = h @ W2 + b2\n\n    loss = F.cross_entropy(logits, torch.tensor(Y))\n    for p in parameters:\n        p.grad = None\n    loss.backward()\n    for p in parameters:\n        p.data += -0.1 * p.grad\n    print(loss.item())<\/code><\/pre>\n\n\n\n
\u8ba9\u6211\u4eec\u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
61.7388916015625\n30.744335174560547\n19.465017318725586\n14.642541885375977\n11.606647491455078\n10.029589653015137\n9.165127754211426\n6.288736343383789\n5.30631160736084\n4.916451930999756\n4.473801612854004\n3.683753252029419\n3.3870129585266113\n3.616569995880127\n5.824367523193359\n6.791852951049805\n3.6995389461517334\n2.742274761199951\n1.8824043273925781\n1.9744515419006348\n1.5300321578979492\n1.6461244821548462\n2.1655921936035156\n3.659118175506592\n2.467254161834717\n1.3796861171722412\n1.0965161323547363\n1.2355493307113647\n1.6014204025268555\n1.3624526262283325\n1.5353899002075195\n1.307451605796814\n1.1866517066955566\n0.8342674374580383\n0.875034749507904\n0.8176398873329163\n1.0066556930541992\n1.0191515684127808\n0.9441428780555725\n0.6796780824661255\n0.693280041217804\n0.7197794914245605\n0.733572781085968\n0.6491024494171143\n0.6743171215057373\n0.6201052069664001\n0.6231799125671387\n0.5758059024810791\n0.5589979887008667\n0.5286083221435547\n0.5157477259635925\n0.49505743384361267\n0.4789549708366394\n0.46755141019821167\n0.4543977975845337\n0.44819700717926025\n0.435586541891098\n0.43247443437576294\n0.4204025864601135\n0.4189830422401428\n0.40780332684516907\n0.4070993661880493\n0.397169828414917\n0.396655797958374\n0.3880215883255005\n0.38747039437294006\n0.37993234395980835\n0.37929850816726685\n0.3726232945919037\n0.37193870544433594\n0.3659595847129822\n0.36525923013687134\n0.3598758280277252\n0.3591753840446472\n0.3543221950531006\n0.35362374782562256\n0.3492506742477417\n0.3485535681247711\n0.3446139693260193\n0.34391850233078003\n0.34036877751350403\n0.3396766483783722\n0.3364769518375397\n0.33578920364379883\n0.33290255069732666\n0.3322204351425171\n0.3296143710613251\n0.3289378881454468\n0.3265831470489502\n0.3259122669696808\n0.32378315925598145\n0.3231176435947418\n0.32119160890579224\n0.32053107023239136\n0.3187873065471649\n0.31813114881515503\n0.3165515661239624\n0.3158996105194092\n0.3144682049751282\n0.31382042169570923<\/code><\/pre>\n\n\n\n
\u53ef\u4ee5\u770b\u5230\u6211\u4eec\u7684\u6837\u672c\u96c6\u53ea\u9009\u7528\u4e86\u524d5\u4e2a\u59d3\u540d\uff0c\u5e76\u4e14\u5bf9\u4e8e\u8fd9\u4e2a\u6a21\u578b\u6765\u8bf4\uff0c\u5b83\u7684\u635f\u5931\u503c\u6b63\u5728\u53d8\u5c0f\u7684\u3002<\/p>\n\n\n\n
\u8fd1\u4f3c\u68af\u5ea6<\/h3>\n\n\n\n
\u6837\u672c\u96c6\u5982\u679c\u8bbe\u5b9a\u7684\u662f\u6240\u6709\u7684\u6837\u672c\u4e00\u8d77\u8ba1\u7b97\u7684\u8bdd\uff0c\u5176\u8ba1\u7b97\u7684\u901f\u5ea6\u4f1a\u5f88\u6162\u3002\u56e0\u6b64\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528\u8fd1\u4f3c\u68af\u5ea6\u7684\u65b9\u5f0f\u52a0\u5feb\u8ba1\u7b97\u8fc7\u7a0b\u3002
\u8fd1\u4f3c\u68af\u5ea6\u5c31\u662f\u5728\u6837\u672c\u96c6\u4e2d\u968f\u673a\u62ff\u53d6\u82e5\u5e72\u7684\u6837\u672c\uff0c\u7ec4\u6210\u4e00\u4e2a\u5c0f\u7684\u6837\u672c\u96c6\uff0c\u5e76\u4e14\u901a\u8fc7\u8fd9\u4e2a\u6837\u672c\u96c6\u53bb\u505a\u524d\u5411\u4f20\u64ad\u548c\u53cd\u5411\u4f20\u64ad\uff0c\u8ba1\u7b97\u68af\u5ea6\u5e76\u8fdb\u884c\u8bad\u7ec3\uff0c\u80fd\u591f\u66f4\u5feb\u5730\u8fbe\u5230\u66f4\u597d\u7684\u8bad\u7ec3\u6548\u679c
\u4f7f\u7528\u8fd1\u4f3c\u68af\u5ea6\u53ef\u4ee5\u6781\u5927\u5730\u52a0\u5feb\u6211\u4eec\u7684\u8bad\u7ec3\u901f\u5ea6\uff0c\u4f46\u662f\u7531\u4e8e\u6bcf\u6b21\u8ba1\u7b97\u51fa\u6765\u7684\u68af\u5ea6\u5e76\u4e0d\u662f\u6574\u4e2a\u6837\u672c\u96c6\u7684\u68af\u5ea6\uff0c\u800c\u662f\u5f53\u524d\u8fd9\u4e2a\u5c0f\u6837\u672c\u7ea7\u7684\u68af\u5ea6\uff0c\u6240\u4ee5\u6211\u4eec\u9700\u8981\u589e\u52a0\u8bad\u7ec3\u6b21\u6570\u6765\u8fbe\u5230\u76f8\u540c\u7684\u6548\u679c
\u6211\u4eec\u5148\u4fee\u6539\u4ee3\u7801\uff0c\u5b9e\u73b0\u8fd1\u4f3c\u68af\u5ea6\u8bad\u7ec3\uff1a<\/p>\n\n\n\n
import torch\nimport torch.nn.functional as F\nimport matplotlib.pyplot as plt\n\nwords = open(\"names.txt\",\"r\").read().splitlines()\ng = torch.Generator().manual_seed(2147483647) # \u56fa\u5b9a\u968f\u673a\u79cd\u5b50\n\n# \u521b\u5efa\u5b57\u5178\u548c\u53cd\u5411\u5b57\u5178\nchars = sorted(list(set(\"\".join(words))))\nstoi = { s : i+1 for i,s in enumerate(chars) } # \u5b57\u7b26\u6620\u5c04\u6574\u6570\u7d22\u5f15\nstoi[\".\"] = 0\nitos = { i : s for s,i in stoi.items() } # \u53cd\u5411\u5b57\u5178\n\nblock_size = 3\nX, Y = [], []\nfor word in words:\n    # \u5148\u5728\u524d\u9762\u52a0\u4e0ablock_size\u4e2a\u201c.\u201d\uff0c\u4e5f\u5c31\u662f\u52a0\u4e0a\u82e5\u5e72\u4e2a0\uff0c\u56e0\u4e3a\u662f\u6309\u7167stoi\u4e2d\u7684\u6570\u5b57\u6765\u4fdd\u5b58\u7684\n    context = [0] * block_size\n    for char in word + \".\":\n        ix = stoi[char]\n        X.append(context)\n        Y.append(ix)\n        # \u5c06context\u5f80\u540e\u632a\u4e00\u4e2a\u5b57\u7b26\n        context = context[1:] + [ix]\n\nC = torch.randn((27,2), generator=g) # \u5d4c\u5165\u5c42\n\n# \u7b2c\u4e00\u5c42\u9690\u85cf\u5c42\nW1 = torch.randn((6,100), generator=g)\nb1 = torch.randn(100, generator=g)\n\n# \u7b2c\u4e8c\u5c42\u8f93\u51fa\u5c42\nW2 = torch.randn((100,27), generator=g)\nb2 = torch.randn(27, generator=g)\n\nparameters = [C,W1,W2,b1,b2]\nX = torch.tensor(X)\nY = torch.tensor(Y)\nfor p in parameters:\n    p.requires_grad = True\n\nfor _ in range(1000):\n\n    ix = torch.randint(0,X.shape[0],(32,))\n    emb = C[X[ix]]  # \u5c06\u6837\u672c\u4e2d\u7684\u5b57\u7b26\u4f9d\u6b21\u8f6c\u5316\u4e3a\u6d6e\u70b9\u6570\u8868\u793a\n\n    h = emb.view(-1, 6) @ W1 + b1\n    logits = h @ W2 + b2\n    loss = F.cross_entropy(logits, Y[ix])\n    for p in parameters:\n        p.grad = None\n    loss.backward()\n    for p in parameters:\n        p.data += -0.1 * p.grad\n    print(loss.item())\n<\/code><\/pre>\n\n\n\n
\u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
49.284637451171875\n42.29495620727539\n32.57741165161133\n31.870040893554688\n21.491657257080078\n17.07381248474121\n16.543869018554688\n......\n2.9590938091278076\n2.8406906127929688\n2.9868624210357666\n2.538074016571045<\/code><\/pre>\n\n\n\n
\u53ef\u4ee5\u770b\u5230\uff0c\u7531\u4e8e\u4f7f\u7528\u4e86\u8fd1\u4f3c\u68af\u5ea6\uff0c\u56e0\u6b64\u6bcf\u6b21\u9009\u62e9\u7684\u5c0f\u6837\u672c\u96c6\u4e5f\u662f\u4e0d\u4e00\u6837\u7684\uff0c\u6240\u4ee5\u6211\u4eec\u4f1a\u53d1\u73b0\u5176\u635f\u5931\u503c\u7684\u53d8\u5316\u8d8b\u52bf\u4e0d\u4e00\u5b9a\u6bcf\u4e00\u6b21\u90fd\u662f\u51cf\u5c11\u7684\u3002\u6211\u4eec\u518d\u589e\u52a0\u4e00\u6bb5\u4ee3\u7801\u6765\u770b\u770b\u6700\u7ec8\u8bad\u7ec3\u7ed3\u679c\u5bf9\u4e8e\u6574\u4f53\u6837\u672c\u7684\u635f\u5931\u503c\uff1a<\/p>\n\n\n\n
emb = C[X]\nh = emb.view(-1, 6) @ W1 + b1\nlogits = h @ W2 + b2\nloss = F.cross_entropy(logits, Y)\nprint(loss.item())<\/code><\/pre>\n\n\n\n
\u8fd0\u884c\u4e00\u4e0b\uff0c\u6211\u5728\u8fd9\u91cc\u5c31\u622a\u53d6\u6700\u540e\u4e24\u6b21\u8f93\u51fa\uff1a<\/p>\n\n\n\n
3.044374465942383\n3.019815444946289<\/code><\/pre>\n\n\n\n
\u53ef\u4ee5\u770b\u5230\uff0c\u5728\u672c\u6b21\u8bad\u7ec3\u4e2d\uff0c\u6700\u540e\u4e00\u6b21\u4f7f\u7528\u76f8\u4f3c\u68af\u5ea6\u8bad\u7ec3\u7684\u635f\u5931\u503c\u662f3.044\uff0c\u800c\u5bf9\u4e8e\u6240\u6709\u6837\u672c\u96c6\u7684\u635f\u5931\u503c\u662f3.019\uff0c\u53ef\u4ee5\u770b\u5230\u5176\u533a\u522b\u4e0d\u662f\u5f88\u5927\uff0c\u4ee3\u8868\u4e86\u6211\u4eec\u7684\u8bad\u7ec3\u8fd8\u662f\u6bd4\u8f83\u6210\u529f\u7684\u3002<\/p>\n\n\n\n
\u6dfb\u52a0\u975e\u7ebf\u6027\u6fc0\u6d3b\u51fd\u6570<\/strong><\/h3>\n\n\n\n
\u4ece\u6570\u5b66\u4e0a\uff0c\u4e24\u4e2a\u7ebf\u6027\u53d8\u6362\u53e0\u52a0\uff0c\u672c\u8d28\u4e0a\u7b49\u4ef7\u4e8e\u4e00\u4e2a\u7ebf\u6027\u53d8\u6362<\/strong>\uff0c\u8fd9\u610f\u5473\u7740\u65e0\u8bba\u6211\u4eec\u7684\u7f51\u7edc\u591a\u6df1\uff0c\u5b83\u90fd\u65e0\u6cd5\u62df\u5408\u975e\u7ebf\u6027\u5173\u7cfb\u3002\u56e0\u6b64\uff0c\u6211\u4eec\u9700\u8981\u4fee\u6539h\uff1a<\/p>\n\n\n\n
h = torch.tanh(emb.view(-1, 6) @ W1 + b1)  # \u52a0\u4e0a tanh \u975e\u7ebf\u6027<\/code><\/pre>\n\n\n\n
\u8ffd\u8e2a\u66f4\u597d\u7684\u5b66\u4e60\u7387<\/h3>\n\n\n\n
\u6211\u4eec\u73b0\u5728\u8fd8\u9700\u8981\u6765\u63a2\u7a76\u4e00\u4e0b\uff0c\u5b66\u4e60\u7387\u8bbe\u7f6e\u4e3a\u591a\u5c11\u6bd4\u8f83\u5408\u9002\u3002\u9996\u5148\u6211\u4eec\u751f\u6210\u4e00\u4e2a-3\u52300\u7684\u7ebf\u6027\u5217\u8868\uff0c\u7136\u540e\u5c06\u5176\u4f5c\u4e3a10\u7684x\u6b21\u5e42\uff0c\u8fd9\u6837\u5c31\u80fd\u83b7\u5f97\u4e00\u4e2a\u6307\u6570\u589e\u957f\u7684\u5217\u8868\uff1a<\/p>\n\n\n\n
# \u5b66\u4e60\u7387\u533a\u95f4\nlre = torch.linspace(-3,0,1000)\nlrs = 10**lre\nprint(lrs)<\/code><\/pre>\n\n\n\n
\u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
tensor([0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0011,\n        0.0011, 0.0011, 0.0011, 0.0011, 0.0011, 0.0011, 0.0011, 0.0011, 0.0011,\n        0.0011, 0.0011, 0.0011, 0.0012, 0.0012, 0.0012, 0.0012, 0.0012, 0.0012,\n        0.0012, 0.0012, 0.0012, 0.0012, 0.0012, 0.0012, 0.0013, 0.0013, 0.0013,\n        0.0013, 0.0013, 0.0013, 0.0013, 0.0013, 0.0013, 0.0013, 0.0013, 0.0014,\n        0.0014, 0.0014, 0.0014, 0.0014, 0.0014, 0.0014, 0.0014, 0.0014, 0.0014,\n        0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015,\n        0.0015, 0.0016, 0.0016, 0.0016, 0.0016, 0.0016, 0.0016, 0.0016, 0.0016,\n        0.0016, 0.0017, 0.0017, 0.0017, 0.0017, 0.0017, 0.0017, 0.0017, 0.0017,\n        0.0018, 0.0018, 0.0018, 0.0018, 0.0018, 0.0018, 0.0018, 0.0018, 0.0019,\n        0.0019, 0.0019, 0.0019, 0.0019, 0.0019, 0.0019, 0.0019, 0.0020, 0.0020,\n        0.0020, 0.0020, 0.0020, 0.0020, 0.0020, 0.0021, 0.0021, 0.0021, 0.0021,\n        0.0021, 0.0021, 0.0021, 0.0022, 0.0022, 0.0022, 0.0022, 0.0022, 0.0022,\n        0.0022, 0.0023, 0.0023, 0.0023, 0.0023, 0.0023, 0.0023, 0.0024, 0.0024,\n        0.0024, 0.0024, 0.0024, 0.0024, 0.0025, 0.0025, 0.0025, 0.0025, 0.0025,\n        0.0025, 0.0026, 0.0026, 0.0026, 0.0026, 0.0026, 0.0027, 0.0027, 0.0027,\n        0.0027, 0.0027, 0.0027, 0.0028, 0.0028, 0.0028, 0.0028, 0.0028, 0.0029,\n        0.0029, 0.0029, 0.0029, 0.0029, 0.0030, 0.0030, 0.0030, 0.0030, 0.0030,\n        0.0031, 0.0031, 0.0031, 0.0031, 0.0032, 0.0032, 0.0032, 0.0032, 0.0032,\n        0.0033, 0.0033, 0.0033, 0.0033, 0.0034, 0.0034, 0.0034, 0.0034, 0.0034,\n        0.0035, 0.0035, 0.0035, 0.0035, 0.0036, 0.0036, 0.0036, 0.0036, 0.0037,\n        0.0037, 0.0037, 0.0037, 0.0038, 0.0038, 0.0038, 0.0039, 0.0039, 0.0039,\n        0.0039, 0.0040, 0.0040, 0.0040, 0.0040, 0.0041, 0.0041, 0.0041, 0.0042,\n        0.0042, 0.0042, 0.0042, 0.0043, 0.0043, 0.0043, 0.0044, 0.0044, 0.0044,\n        0.0045, 0.0045, 0.0045, 0.0045, 0.0046, 0.0046, 0.0046, 0.0047, 0.0047,\n        0.0047, 0.0048, 0.0048, 0.0048, 0.0049, 0.0049, 0.0049, 0.0050, 0.0050,\n        0.0050, 0.0051, 0.0051, 0.0051, 0.0052, 0.0052, 0.0053, 0.0053, 0.0053,\n        0.0054, 0.0054, 0.0054, 0.0055, 0.0055, 0.0056, 0.0056, 0.0056, 0.0057,\n        0.0057, 0.0058, 0.0058, 0.0058, 0.0059, 0.0059, 0.0060, 0.0060, 0.0060,\n        0.0061, 0.0061, 0.0062, 0.0062, 0.0062, 0.0063, 0.0063, 0.0064, 0.0064,\n        0.0065, 0.0065, 0.0066, 0.0066, 0.0067, 0.0067, 0.0067, 0.0068, 0.0068,\n        0.0069, 0.0069, 0.0070, 0.0070, 0.0071, 0.0071, 0.0072, 0.0072, 0.0073,\n        0.0073, 0.0074, 0.0074, 0.0075, 0.0075, 0.0076, 0.0076, 0.0077, 0.0077,\n        0.0078, 0.0079, 0.0079, 0.0080, 0.0080, 0.0081, 0.0081, 0.0082, 0.0082,\n        0.0083, 0.0084, 0.0084, 0.0085, 0.0085, 0.0086, 0.0086, 0.0087, 0.0088,\n        0.0088, 0.0089, 0.0090, 0.0090, 0.0091, 0.0091, 0.0092, 0.0093, 0.0093,\n        0.0094, 0.0095, 0.0095, 0.0096, 0.0097, 0.0097, 0.0098, 0.0099, 0.0099,\n        0.0100, 0.0101, 0.0101, 0.0102, 0.0103, 0.0104, 0.0104, 0.0105, 0.0106,\n        0.0106, 0.0107, 0.0108, 0.0109, 0.0109, 0.0110, 0.0111, 0.0112, 0.0112,\n        0.0113, 0.0114, 0.0115, 0.0116, 0.0116, 0.0117, 0.0118, 0.0119, 0.0120,\n        0.0121, 0.0121, 0.0122, 0.0123, 0.0124, 0.0125, 0.0126, 0.0127, 0.0127,\n        0.0128, 0.0129, 0.0130, 0.0131, 0.0132, 0.0133, 0.0134, 0.0135, 0.0136,\n        0.0137, 0.0137, 0.0138, 0.0139, 0.0140, 0.0141, 0.0142, 0.0143, 0.0144,\n        0.0145, 0.0146, 0.0147, 0.0148, 0.0149, 0.0150, 0.0151, 0.0152, 0.0154,\n        0.0155, 0.0156, 0.0157, 0.0158, 0.0159, 0.0160, 0.0161, 0.0162, 0.0163,\n        0.0165, 0.0166, 0.0167, 0.0168, 0.0169, 0.0170, 0.0171, 0.0173, 0.0174,\n        0.0175, 0.0176, 0.0178, 0.0179, 0.0180, 0.0181, 0.0182, 0.0184, 0.0185,\n        0.0186, 0.0188, 0.0189, 0.0190, 0.0192, 0.0193, 0.0194, 0.0196, 0.0197,\n        0.0198, 0.0200, 0.0201, 0.0202, 0.0204, 0.0205, 0.0207, 0.0208, 0.0210,\n        0.0211, 0.0212, 0.0214, 0.0215, 0.0217, 0.0218, 0.0220, 0.0221, 0.0223,\n        0.0225, 0.0226, 0.0228, 0.0229, 0.0231, 0.0232, 0.0234, 0.0236, 0.0237,\n        0.0239, 0.0241, 0.0242, 0.0244, 0.0246, 0.0247, 0.0249, 0.0251, 0.0253,\n        0.0254, 0.0256, 0.0258, 0.0260, 0.0261, 0.0263, 0.0265, 0.0267, 0.0269,\n        0.0271, 0.0273, 0.0274, 0.0276, 0.0278, 0.0280, 0.0282, 0.0284, 0.0286,\n        0.0288, 0.0290, 0.0292, 0.0294, 0.0296, 0.0298, 0.0300, 0.0302, 0.0304,\n        0.0307, 0.0309, 0.0311, 0.0313, 0.0315, 0.0317, 0.0320, 0.0322, 0.0324,\n        0.0326, 0.0328, 0.0331, 0.0333, 0.0335, 0.0338, 0.0340, 0.0342, 0.0345,\n        0.0347, 0.0350, 0.0352, 0.0354, 0.0357, 0.0359, 0.0362, 0.0364, 0.0367,\n        0.0369, 0.0372, 0.0375, 0.0377, 0.0380, 0.0382, 0.0385, 0.0388, 0.0390,\n        0.0393, 0.0396, 0.0399, 0.0401, 0.0404, 0.0407, 0.0410, 0.0413, 0.0416,\n        0.0418, 0.0421, 0.0424, 0.0427, 0.0430, 0.0433, 0.0436, 0.0439, 0.0442,\n        0.0445, 0.0448, 0.0451, 0.0455, 0.0458, 0.0461, 0.0464, 0.0467, 0.0471,\n        0.0474, 0.0477, 0.0480, 0.0484, 0.0487, 0.0491, 0.0494, 0.0497, 0.0501,\n        0.0504, 0.0508, 0.0511, 0.0515, 0.0518, 0.0522, 0.0526, 0.0529, 0.0533,\n        0.0537, 0.0540, 0.0544, 0.0548, 0.0552, 0.0556, 0.0559, 0.0563, 0.0567,\n        0.0571, 0.0575, 0.0579, 0.0583, 0.0587, 0.0591, 0.0595, 0.0599, 0.0604,\n        0.0608, 0.0612, 0.0616, 0.0621, 0.0625, 0.0629, 0.0634, 0.0638, 0.0642,\n        0.0647, 0.0651, 0.0656, 0.0660, 0.0665, 0.0670, 0.0674, 0.0679, 0.0684,\n        0.0688, 0.0693, 0.0698, 0.0703, 0.0708, 0.0713, 0.0718, 0.0723, 0.0728,\n        0.0733, 0.0738, 0.0743, 0.0748, 0.0753, 0.0758, 0.0764, 0.0769, 0.0774,\n        0.0780, 0.0785, 0.0790, 0.0796, 0.0802, 0.0807, 0.0813, 0.0818, 0.0824,\n        0.0830, 0.0835, 0.0841, 0.0847, 0.0853, 0.0859, 0.0865, 0.0871, 0.0877,\n        0.0883, 0.0889, 0.0895, 0.0901, 0.0908, 0.0914, 0.0920, 0.0927, 0.0933,\n        0.0940, 0.0946, 0.0953, 0.0959, 0.0966, 0.0973, 0.0979, 0.0986, 0.0993,\n        0.1000, 0.1007, 0.1014, 0.1021, 0.1028, 0.1035, 0.1042, 0.1050, 0.1057,\n        0.1064, 0.1072, 0.1079, 0.1087, 0.1094, 0.1102, 0.1109, 0.1117, 0.1125,\n        0.1133, 0.1140, 0.1148, 0.1156, 0.1164, 0.1172, 0.1181, 0.1189, 0.1197,\n        0.1205, 0.1214, 0.1222, 0.1231, 0.1239, 0.1248, 0.1256, 0.1265, 0.1274,\n        0.1283, 0.1292, 0.1301, 0.1310, 0.1319, 0.1328, 0.1337, 0.1346, 0.1356,\n        0.1365, 0.1374, 0.1384, 0.1394, 0.1403, 0.1413, 0.1423, 0.1433, 0.1443,\n        0.1453, 0.1463, 0.1473, 0.1483, 0.1493, 0.1504, 0.1514, 0.1525, 0.1535,\n        0.1546, 0.1557, 0.1567, 0.1578, 0.1589, 0.1600, 0.1611, 0.1623, 0.1634,\n        0.1645, 0.1657, 0.1668, 0.1680, 0.1691, 0.1703, 0.1715, 0.1727, 0.1739,\n        0.1751, 0.1763, 0.1775, 0.1788, 0.1800, 0.1812, 0.1825, 0.1838, 0.1850,\n        0.1863, 0.1876, 0.1889, 0.1902, 0.1916, 0.1929, 0.1942, 0.1956, 0.1969,\n        0.1983, 0.1997, 0.2010, 0.2024, 0.2038, 0.2053, 0.2067, 0.2081, 0.2096,\n        0.2110, 0.2125, 0.2140, 0.2154, 0.2169, 0.2184, 0.2200, 0.2215, 0.2230,\n        0.2246, 0.2261, 0.2277, 0.2293, 0.2309, 0.2325, 0.2341, 0.2357, 0.2373,\n        0.2390, 0.2406, 0.2423, 0.2440, 0.2457, 0.2474, 0.2491, 0.2508, 0.2526,\n        0.2543, 0.2561, 0.2579, 0.2597, 0.2615, 0.2633, 0.2651, 0.2669, 0.2688,\n        0.2707, 0.2725, 0.2744, 0.2763, 0.2783, 0.2802, 0.2821, 0.2841, 0.2861,\n        0.2880, 0.2900, 0.2921, 0.2941, 0.2961, 0.2982, 0.3002, 0.3023, 0.3044,\n        0.3065, 0.3087, 0.3108, 0.3130, 0.3151, 0.3173, 0.3195, 0.3217, 0.3240,\n        0.3262, 0.3285, 0.3308, 0.3331, 0.3354, 0.3377, 0.3400, 0.3424, 0.3448,\n        0.3472, 0.3496, 0.3520, 0.3544, 0.3569, 0.3594, 0.3619, 0.3644, 0.3669,\n        0.3695, 0.3720, 0.3746, 0.3772, 0.3798, 0.3825, 0.3851, 0.3878, 0.3905,\n        0.3932, 0.3959, 0.3987, 0.4014, 0.4042, 0.4070, 0.4098, 0.4127, 0.4155,\n        0.4184, 0.4213, 0.4243, 0.4272, 0.4302, 0.4331, 0.4362, 0.4392, 0.4422,\n        0.4453, 0.4484, 0.4515, 0.4546, 0.4578, 0.4610, 0.4642, 0.4674, 0.4706,\n        0.4739, 0.4772, 0.4805, 0.4838, 0.4872, 0.4906, 0.4940, 0.4974, 0.5008,\n        0.5043, 0.5078, 0.5113, 0.5149, 0.5185, 0.5221, 0.5257, 0.5293, 0.5330,\n        0.5367, 0.5404, 0.5442, 0.5479, 0.5517, 0.5556, 0.5594, 0.5633, 0.5672,\n        0.5712, 0.5751, 0.5791, 0.5831, 0.5872, 0.5913, 0.5954, 0.5995, 0.6036,\n        0.6078, 0.6120, 0.6163, 0.6206, 0.6249, 0.6292, 0.6336, 0.6380, 0.6424,\n        0.6469, 0.6513, 0.6559, 0.6604, 0.6650, 0.6696, 0.6743, 0.6789, 0.6837,\n        0.6884, 0.6932, 0.6980, 0.7028, 0.7077, 0.7126, 0.7176, 0.7225, 0.7275,\n        0.7326, 0.7377, 0.7428, 0.7480, 0.7531, 0.7584, 0.7636, 0.7689, 0.7743,\n        0.7796, 0.7850, 0.7905, 0.7960, 0.8015, 0.8071, 0.8127, 0.8183, 0.8240,\n        0.8297, 0.8355, 0.8412, 0.8471, 0.8530, 0.8589, 0.8648, 0.8708, 0.8769,\n        0.8830, 0.8891, 0.8953, 0.9015, 0.9077, 0.9140, 0.9204, 0.9268, 0.9332,\n        0.9397, 0.9462, 0.9528, 0.9594, 0.9660, 0.9727, 0.9795, 0.9863, 0.9931,\n        1.0000])<\/code><\/pre>\n\n\n\n
\u63a5\u4e0b\u6765\u6211\u4eec\u4fee\u6539\u4ee3\u7801\uff0c\u4fdd\u5b58\u4e0d\u540c\u7684\u5b66\u4e60\u7387\u5bf9\u635f\u5931\u9020\u6210\u7684\u5f71\u54cd\uff1a<\/p>\n\n\n\n
import torch\nimport torch.nn.functional as F\nimport matplotlib.pyplot as plt\n\nwords = open(\"names.txt\",\"r\").read().splitlines()\ng = torch.Generator().manual_seed(2147483647) # \u56fa\u5b9a\u968f\u673a\u79cd\u5b50\n\n# \u521b\u5efa\u5b57\u5178\u548c\u53cd\u5411\u5b57\u5178\nchars = sorted(list(set(\"\".join(words))))\nstoi = { s : i+1 for i,s in enumerate(chars) } # \u5b57\u7b26\u6620\u5c04\u6574\u6570\u7d22\u5f15\nstoi[\".\"] = 0\nitos = { i : s for s,i in stoi.items() } # \u53cd\u5411\u5b57\u5178\n\nblock_size = 3\nX, Y = [], []\nfor word in words:\n    # \u5148\u5728\u524d\u9762\u52a0\u4e0ablock_size\u4e2a\u201c.\u201d\uff0c\u4e5f\u5c31\u662f\u52a0\u4e0a\u82e5\u5e72\u4e2a0\uff0c\u56e0\u4e3a\u662f\u6309\u7167stoi\u4e2d\u7684\u6570\u5b57\u6765\u4fdd\u5b58\u7684\n    context = [0] * block_size\n    for char in word + \".\":\n        ix = stoi[char]\n        X.append(context)\n        Y.append(ix)\n        # \u5c06context\u5f80\u540e\u632a\u4e00\u4e2a\u5b57\u7b26\n        context = context[1:] + [ix]\n\nC = torch.randn((27,2), generator=g) # \u5d4c\u5165\u5c42\n\n# \u7b2c\u4e00\u5c42\u9690\u85cf\u5c42\nW1 = torch.randn((6,100), generator=g)\nb1 = torch.randn(100, generator=g)\n\n# \u7b2c\u4e8c\u5c42\u8f93\u51fa\u5c42\nW2 = torch.randn((100,27), generator=g)\nb2 = torch.randn(27, generator=g)\n\nparameters = [C,W1,W2,b1,b2]\nX = torch.tensor(X)\nY = torch.tensor(Y)\n\n# \u5b66\u4e60\u7387\u533a\u95f4\nlre = torch.linspace(-3,0,1000)\nlrs = 10**lre\n# print(lrs)\n\nfor p in parameters:\n    p.requires_grad = True\n\nlri = []\nlossi = []\n\nfor i in range(1000):\n\n    ix = torch.randint(0,X.shape[0],(32,))\n    emb = C[X[ix]]  # \u5c06\u6837\u672c\u4e2d\u7684\u5b57\u7b26\u4f9d\u6b21\u8f6c\u5316\u4e3a\u6d6e\u70b9\u6570\u8868\u793a\n\n    h = torch.tanh(emb.view(-1, 6) @ W1 + b1)\n    logits = h @ W2 + b2\n    loss = F.cross_entropy(logits, Y[ix])\n    for p in parameters:\n        p.grad = None\n    loss.backward()\n\n    lr = lrs[i]\n    for p in parameters:\n        p.data += -lr * p.grad\n\n    lri.append(lre[i])\n    lossi.append(loss.item())\n    # print(loss.item())\n\nemb = C[X]\nh = torch.tanh(emb.view(-1, 6) @ W1 + b1)\nlogits = h @ W2 + b2\nloss = F.cross_entropy(logits, Y)\n# print(loss.item())\n\nplt.plot(lri, lossi)\nplt.xlabel('Learning Rate')\nplt.ylabel('Loss')\nplt.show()<\/code><\/pre>\n\n\n\n
\u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
\"\"<\/div><\/figure>\n\n\n\n
\u6211\u4eec\u53ef\u4ee5\u770b\u5230\uff0c\u5b66\u4e60\u7387\u5728-1\u5904\u6bd4\u8f83\u4f18\u79c0\uff0c\u4e5f\u5c31\u662f10**-1=0.1\u3002\u4e5f\u5c31\u662f\u8bf4\uff0c\u6211\u4eec\u6700\u540e\u5f97\u51fa\u7ed3\u8bba\uff0c\u5b66\u4e60\u7387\u8bbe\u4e3a0.1\u6bd4\u8f83\u597d<\/p>\n\n\n\n
\u5b66\u4e60\u7387\u8870\u51cf<\/h3>\n\n\n\n
\u56e0\u4e3a\u4e00\u5f00\u59cb\u4f7f\u75280.1\u7684\u5b66\u4e60\u7387\u53ef\u4ee5\u83b7\u5f97\u8f83\u597d\u7684\u6548\u679c\uff0c\u4f46\u5f53\u8bad\u7ec3\u6b21\u6570\u66f4\u5927\uff0c\u6a21\u578b\u5df2\u7ecf\u53d8\u5f97\u66f4\u7cbe\u7ec6\u7684\u65f6\u5019\uff0c\u6211\u4eec\u53ef\u4ee5\u5bf9\u5b66\u4e60\u7387\u8fdb\u884c\u8870\u51cf\uff0c\u6bd4\u5982\u8870\u51cf\u5176\u4e3a1\/10\uff0c\u8fd9\u6837\u53ef\u4ee5\u8ba9\u6211\u4eec\u7684\u6a21\u578b\u8fbe\u5230\u66f4\u597d\u7684\u6548\u679c\uff1a<\/p>\n\n\n\n
import torch\nimport torch.nn.functional as F\nimport matplotlib.pyplot as plt\n\nwords = open(\"names.txt\",\"r\").read().splitlines()\ng = torch.Generator().manual_seed(2147483647) # \u56fa\u5b9a\u968f\u673a\u79cd\u5b50\n\n# \u521b\u5efa\u5b57\u5178\u548c\u53cd\u5411\u5b57\u5178\nchars = sorted(list(set(\"\".join(words))))\nstoi = { s : i+1 for i,s in enumerate(chars) } # \u5b57\u7b26\u6620\u5c04\u6574\u6570\u7d22\u5f15\nstoi[\".\"] = 0\nitos = { i : s for s,i in stoi.items() } # \u53cd\u5411\u5b57\u5178\n\nblock_size = 3\nX, Y = [], []\nfor word in words:\n    # \u5148\u5728\u524d\u9762\u52a0\u4e0ablock_size\u4e2a\u201c.\u201d\uff0c\u4e5f\u5c31\u662f\u52a0\u4e0a\u82e5\u5e72\u4e2a0\uff0c\u56e0\u4e3a\u662f\u6309\u7167stoi\u4e2d\u7684\u6570\u5b57\u6765\u4fdd\u5b58\u7684\n    context = [0] * block_size\n    for char in word + \".\":\n        ix = stoi[char]\n        X.append(context)\n        Y.append(ix)\n        # \u5c06context\u5f80\u540e\u632a\u4e00\u4e2a\u5b57\u7b26\n        context = context[1:] + [ix]\n\nC = torch.randn((27,2), generator=g) # \u5d4c\u5165\u5c42\n\n# \u7b2c\u4e00\u5c42\u9690\u85cf\u5c42\nW1 = torch.randn((6,100), generator=g)\nb1 = torch.randn(100, generator=g)\n\n# \u7b2c\u4e8c\u5c42\u8f93\u51fa\u5c42\nW2 = torch.randn((100,27), generator=g)\nb2 = torch.randn(27, generator=g)\n\nparameters = [C,W1,W2,b1,b2]\nX = torch.tensor(X)\nY = torch.tensor(Y)\n\nfor p in parameters:\n    p.requires_grad = True\n\nfor i in range(200000):\n\n    ix = torch.randint(0,X.shape[0],(32,))\n    emb = C[X[ix]]  # \u5c06\u6837\u672c\u4e2d\u7684\u5b57\u7b26\u4f9d\u6b21\u8f6c\u5316\u4e3a\u6d6e\u70b9\u6570\u8868\u793a\n\n    h = torch.tanh(emb.view(-1, 6) @ W1 + b1)\n    logits = h @ W2 + b2\n    loss = F.cross_entropy(logits, Y[ix])\n    for p in parameters:\n        p.grad = None\n    loss.backward()\n\n    lr = 0.1 if i < 100000 else 0.01\n    for p in parameters:\n        p.data += -lr * p.grad\n\n    if i % 10000 == 0:\n        print(loss.item())\n\nemb = C[X]\nh = torch.tanh(emb.view(-1, 6) @ W1 + b1)\nlogits = h @ W2 + b2\nloss = F.cross_entropy(logits, Y)\nprint(loss.item())<\/code><\/pre>\n\n\n\n
\u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
18.17679786682129\n2.4790353775024414\n2.3036465644836426\n2.5743117332458496\n2.6201894283294678\n2.1168222427368164\n2.4747679233551025\n2.5844204425811768\n2.0612902641296387\n2.0741043090820312\n2.733124256134033\n2.4877028465270996\n2.463047504425049\n2.0046935081481934\n2.7823586463928223\n2.194141387939453\n1.9175939559936523\n2.2526872158050537\n2.0886881351470947\n2.3008806705474854\n2.2558350563049316<\/code><\/pre>\n\n\n\n
\u62c6\u5206\u6837\u672c\u96c6<\/h3>\n\n\n\n
\u6211\u4eec\u53ef\u4ee5\u628a\u6574\u4e2a\u6837\u672c\u96c6\u62c6\u5206\u6210\u8bad\u7ec3\u96c6\u3001\u9a8c\u8bc1\u96c6\u548c\u6d4b\u8bd5\u96c6\uff0c\u5206\u522b\u5360\u6bd480%10%\u548c10%\u3002\u73b0\u5728\u8ba9\u6211\u4eec\u6765\u5b9e\u73b0\u4e00\u4e0b\uff0c\u6211\u4eec\u7528\u8bad\u7ec3\u96c6\u6765\u8bad\u7ec3\uff0c\u7528\u9a8c\u8bc1\u96c6\u6765\u68c0\u6d4b\u6700\u7ec8\u7684\u7ed3\u679c\u3002\u7531\u4e8e\u6211\u4eec\u76ee\u524d\u5b9e\u73b0\u7684\u6bd4\u8f83\u7c97\u7cd9\uff0c\u6ca1\u6709\u901a\u8fc7\u9a8c\u8bc1\u96c6\u6765\u51b3\u5b9a\u5b66\u4e60\u7387\u7684\u9ad8\u4f4e\uff0c\u6d4b\u8bd5\u96c6\u53cd\u800c\u6ca1\u6709\u7528\u4e86\u3002\u6211\u4eec\u6765\u5b9e\u73b0\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
import torch\nimport torch.nn.functional as F\nimport matplotlib.pyplot as plt\nimport random\n\nwords = open(\"names.txt\",\"r\").read().splitlines()\ng = torch.Generator().manual_seed(2147483647) # \u56fa\u5b9a\u968f\u673a\u79cd\u5b50\n\n# \u521b\u5efa\u5b57\u5178\u548c\u53cd\u5411\u5b57\u5178\nchars = sorted(list(set(\"\".join(words))))\nstoi = { s : i+1 for i,s in enumerate(chars) } # \u5b57\u7b26\u6620\u5c04\u6574\u6570\u7d22\u5f15\nstoi[\".\"] = 0\nitos = { i : s for s,i in stoi.items() } # \u53cd\u5411\u5b57\u5178\n\ndef build_dataset(words):\n    block_size = 3\n    X, Y = [], []\n    for word in words:\n        # \u5148\u5728\u524d\u9762\u52a0\u4e0ablock_size\u4e2a\u201c.\u201d\uff0c\u4e5f\u5c31\u662f\u52a0\u4e0a\u82e5\u5e72\u4e2a0\uff0c\u56e0\u4e3a\u662f\u6309\u7167stoi\u4e2d\u7684\u6570\u5b57\u6765\u4fdd\u5b58\u7684\n        context = [0] * block_size\n        for char in word + \".\":\n            ix = stoi[char]\n            X.append(context)\n            Y.append(ix)\n            # \u5c06context\u5f80\u540e\u632a\u4e00\u4e2a\u5b57\u7b26\n            context = context[1:] + [ix]\n    X = torch.tensor(X)\n    Y = torch.tensor(Y)\n    print(X.shape, Y.shape)\n    return X, Y\n\nrandom.seed(42)\nrandom.shuffle(words)\nn1 = int(0.8*len(words))\nn2 = int(0.9*len(words))\nXtr, Ytr = build_dataset(words[:n1])\nXdev, Ydev = build_dataset(words[n1:n2])\nXte, Yte = build_dataset(words[n2:])\n\n\nC = torch.randn((27,2), generator=g) # \u5d4c\u5165\u5c42\n\n# \u7b2c\u4e00\u5c42\u9690\u85cf\u5c42\nW1 = torch.randn((6,100), generator=g)\nb1 = torch.randn(100, generator=g)\n\n# \u7b2c\u4e8c\u5c42\u8f93\u51fa\u5c42\nW2 = torch.randn((100,27), generator=g)\nb2 = torch.randn(27, generator=g)\n\nparameters = [C,W1,W2,b1,b2]\n\n\nfor p in parameters:\n    p.requires_grad = True\n\nfor i in range(200000):\n\n    ix = torch.randint(0,Xtr.shape[0],(32,))\n    emb = C[Xtr[ix]]  # \u5c06\u6837\u672c\u4e2d\u7684\u5b57\u7b26\u4f9d\u6b21\u8f6c\u5316\u4e3a\u6d6e\u70b9\u6570\u8868\u793a\n\n    h = torch.tanh(emb.view(-1, 6) @ W1 + b1)\n    logits = h @ W2 + b2\n    loss = F.cross_entropy(logits, Ytr[ix])\n    for p in parameters:\n        p.grad = None\n    loss.backward()\n\n    lr = 0.1 if i < 100000 else 0.01\n    for p in parameters:\n        p.data += -lr * p.grad\n\n    if i % 10000 == 0:\n        print(loss.item())\n\nemb = C[Xdev]\nh = torch.tanh(emb.view(-1, 6) @ W1 + b1)\nlogits = h @ W2 + b2\nloss = F.cross_entropy(logits, Ydev)\nprint(loss.item())<\/code><\/pre>\n\n\n\n
\u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
torch.Size([182625, 3]) torch.Size([182625])\ntorch.Size([22655, 3]) torch.Size([22655])\ntorch.Size([22866, 3]) torch.Size([22866])\n20.83146095275879\n2.6087646484375\n2.931648015975952\n2.515237808227539\n2.7970528602600098\n2.529646396636963\n2.2188212871551514\n2.567169189453125\n2.2405734062194824\n2.0721330642700195\n2.15317440032959\n2.5734517574310303\n1.836839199066162\n2.461702346801758\n2.326200008392334\n2.4253039360046387\n2.2661197185516357\n1.8648080825805664\n2.230231523513794\n2.241915225982666\n2.2516772747039795<\/code><\/pre>\n\n\n\n
\u53ef\u4ee5\u770b\u5230\uff0c\u5728\u8bad\u7ec3\u96c6\u4e2d\u8bad\u7ec3\u7684\u6700\u540e\u4e00\u6b21\u635f\u5931\u662f2.2419\uff0c\u800c\u5728\u9a8c\u8bc1\u96c6\u4e0a\u8fdb\u884c\u9a8c\u8bc1\u7684\u635f\u5931\u503c\u662f2.2516\uff0c\u8fd8\u662f\u6bd4\u8f83\u63a5\u8fd1\u7684\u3002<\/p>\n\n\n\n
\u4f18\u5316\u635f\u5931\u503c<\/h3>\n\n\n\n
\u76ee\u524d\u635f\u5931\u503c\u88ab\u6211\u4eec\u538b\u5230\u4e862.25\u5de6\u53f3\u3002\u6211\u4eec\u53ef\u4ee5\u901a\u8fc7\u589e\u5927\u5d4c\u5165\u5c42\u7684\u7ef4\u6570\uff0c\u548c\u589e\u52a0\u4e2d\u95f4\u9690\u85cf\u5c42\u7684\u795e\u7ecf\u5143\u6570\u91cf\uff0c\u8ba9\u6211\u4eec\u7684\u6a21\u578b\u7684\u635f\u5931\u503c\u66f4\u52a0\u4f4e\uff1a<\/p>\n\n\n\n
import torch\nimport torch.nn.functional as F\nimport matplotlib.pyplot as plt\nimport random\n\nwords = open(\"names.txt\",\"r\").read().splitlines()\ng = torch.Generator().manual_seed(2147483647) # \u56fa\u5b9a\u968f\u673a\u79cd\u5b50\n\n# \u521b\u5efa\u5b57\u5178\u548c\u53cd\u5411\u5b57\u5178\nchars = sorted(list(set(\"\".join(words))))\nstoi = { s : i+1 for i,s in enumerate(chars) } # \u5b57\u7b26\u6620\u5c04\u6574\u6570\u7d22\u5f15\nstoi[\".\"] = 0\nitos = { i : s for s,i in stoi.items() } # \u53cd\u5411\u5b57\u5178\nblock_size = 3\n\ndef build_dataset(words):\n    X, Y = [], []\n    for word in words:\n        # \u5148\u5728\u524d\u9762\u52a0\u4e0ablock_size\u4e2a\u201c.\u201d\uff0c\u4e5f\u5c31\u662f\u52a0\u4e0a\u82e5\u5e72\u4e2a0\uff0c\u56e0\u4e3a\u662f\u6309\u7167stoi\u4e2d\u7684\u6570\u5b57\u6765\u4fdd\u5b58\u7684\n        context = [0] * block_size\n        for char in word + \".\":\n            ix = stoi[char]\n            X.append(context)\n            Y.append(ix)\n            # \u5c06context\u5f80\u540e\u632a\u4e00\u4e2a\u5b57\u7b26\n            context = context[1:] + [ix]\n    X = torch.tensor(X)\n    Y = torch.tensor(Y)\n    print(X.shape, Y.shape)\n    return X, Y\n\nrandom.seed(42)\nrandom.shuffle(words)\nn1 = int(0.8*len(words))\nn2 = int(0.9*len(words))\nXtr, Ytr = build_dataset(words[:n1])\nXdev, Ydev = build_dataset(words[n1:n2])\nXte, Yte = build_dataset(words[n2:])\n\nc_size = 30\nC = torch.randn((27,c_size), generator=g) # \u5d4c\u5165\u5c42\n\n# \u7b2c\u4e00\u5c42\u9690\u85cf\u5c42\nW1 = torch.randn((c_size*block_size,500), generator=g)\nb1 = torch.randn(500, generator=g)\n\n# \u7b2c\u4e8c\u5c42\u8f93\u51fa\u5c42\nW2 = torch.randn((500,27), generator=g)\nb2 = torch.randn(27, generator=g)\n\nparameters = [C,W1,W2,b1,b2]\n\n\nfor p in parameters:\n    p.requires_grad = True\n\nfor i in range(200000):\n\n    ix = torch.randint(0,Xtr.shape[0],(32,))\n    emb = C[Xtr[ix]]  # \u5c06\u6837\u672c\u4e2d\u7684\u5b57\u7b26\u4f9d\u6b21\u8f6c\u5316\u4e3a\u6d6e\u70b9\u6570\u8868\u793a\n\n    h = torch.tanh(emb.view(-1, c_size*block_size) @ W1 + b1)\n    logits = h @ W2 + b2\n    loss = F.cross_entropy(logits, Ytr[ix])\n    for p in parameters:\n        p.grad = None\n    loss.backward()\n\n    lr = 0.1 if i < 100000 else 0.01\n    for p in parameters:\n        p.data += -lr * p.grad\n\n    if i % 10000 == 0:\n        print(loss.item())\n\nemb = C[Xdev]\nh = torch.tanh(emb.view(-1, c_size*block_size) @ W1 + b1)\nlogits = h @ W2 + b2\nloss = F.cross_entropy(logits, Ydev)\nprint(loss.item())<\/code><\/pre>\n\n\n\n
\u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
D:\\Data\\Anacondea\\envs\\lora\\python.exe D:\\Data\\Python\\Class_3\\main.py \ntorch.Size([182625, 3]) torch.Size([182625])\ntorch.Size([22655, 3]) torch.Size([22655])\ntorch.Size([22866, 3]) torch.Size([22866])\n41.01119613647461\n2.6473662853240967\n3.1581287384033203\n2.8194284439086914\n3.114468574523926\n2.694441795349121\n2.2945523262023926\n2.6995344161987305\n2.6593563556671143\n3.0314278602600098\n2.737340211868286\n1.6961545944213867\n1.9784475564956665\n2.1313881874084473\n2.1206936836242676\n1.9203801155090332\n2.08767032623291\n1.915178894996643\n2.2067370414733887\n1.9190465211868286\n2.1798207759857178<\/code><\/pre>\n\n\n\n
\u53ef\u4ee5\u770b\u5230\uff0c\u6211\u4eec\u7684\u635f\u5931\u503c\u5df2\u7ecf\u88ab\u964d\u52302.1798\u4e86\uff0c\u751a\u81f3\u5f88\u591a\u65f6\u5019\u80fd\u964d\u52302\u4ee5\u4e0b\u3002<\/p>\n\n\n\n
\u8ba9\u6a21\u578b\u8f93\u51fa<\/h3>\n\n\n\n
\u6700\u540e\u6211\u4eec\u5b9e\u73b0\u4e00\u4e2a\u65b9\u6cd5\uff0c\u8ba9\u8bad\u7ec3\u597d\u7684\u6a21\u578b\u8fdb\u884c\u8f93\u51fa\uff0c\u4ee5\u4e0b\u662f\u6700\u7ec8\u5b8c\u6574\u4ee3\u7801\uff1a<\/p>\n\n\n\n
import torch\nimport torch.nn.functional as F\nimport matplotlib.pyplot as plt\nimport random\n\nwords = open(\"names.txt\",\"r\").read().splitlines()\ng = torch.Generator().manual_seed(2147483647) # \u56fa\u5b9a\u968f\u673a\u79cd\u5b50\n\n# \u521b\u5efa\u5b57\u5178\u548c\u53cd\u5411\u5b57\u5178\nchars = sorted(list(set(\"\".join(words))))\nstoi = { s : i+1 for i,s in enumerate(chars) } # \u5b57\u7b26\u6620\u5c04\u6574\u6570\u7d22\u5f15\nstoi[\".\"] = 0\nitos = { i : s for s,i in stoi.items() } # \u53cd\u5411\u5b57\u5178\nblock_size = 3\n\ndef build_dataset(words):\n    X, Y = [], []\n    for word in words:\n        # \u5148\u5728\u524d\u9762\u52a0\u4e0ablock_size\u4e2a\u201c.\u201d\uff0c\u4e5f\u5c31\u662f\u52a0\u4e0a\u82e5\u5e72\u4e2a0\uff0c\u56e0\u4e3a\u662f\u6309\u7167stoi\u4e2d\u7684\u6570\u5b57\u6765\u4fdd\u5b58\u7684\n        context = [0] * block_size\n        for char in word + \".\":\n            ix = stoi[char]\n            X.append(context)\n            Y.append(ix)\n            # \u5c06context\u5f80\u540e\u632a\u4e00\u4e2a\u5b57\u7b26\n            context = context[1:] + [ix]\n    X = torch.tensor(X)\n    Y = torch.tensor(Y)\n    print(X.shape, Y.shape)\n    return X, Y\n\nrandom.seed(42)\nrandom.shuffle(words)\nn1 = int(0.8*len(words))\nn2 = int(0.9*len(words))\nXtr, Ytr = build_dataset(words[:n1])\nXdev, Ydev = build_dataset(words[n1:n2])\nXte, Yte = build_dataset(words[n2:])\n\nc_size = 30\nC = torch.randn((27,c_size), generator=g) # \u5d4c\u5165\u5c42\n\n# \u7b2c\u4e00\u5c42\u9690\u85cf\u5c42\nW1 = torch.randn((c_size*block_size,500), generator=g)\nb1 = torch.randn(500, generator=g)\n\n# \u7b2c\u4e8c\u5c42\u8f93\u51fa\u5c42\nW2 = torch.randn((500,27), generator=g)\nb2 = torch.randn(27, generator=g)\n\nparameters = [C,W1,W2,b1,b2]\n\n\nfor p in parameters:\n    p.requires_grad = True\n\nfor i in range(200000):\n\n    ix = torch.randint(0,Xtr.shape[0],(32,))\n    emb = C[Xtr[ix]]  # \u5c06\u6837\u672c\u4e2d\u7684\u5b57\u7b26\u4f9d\u6b21\u8f6c\u5316\u4e3a\u6d6e\u70b9\u6570\u8868\u793a\n\n    h = torch.tanh(emb.view(-1, c_size*block_size) @ W1 + b1)\n    logits = h @ W2 + b2\n    loss = F.cross_entropy(logits, Ytr[ix])\n    for p in parameters:\n        p.grad = None\n    loss.backward()\n\n    lr = 0.1 if i < 100000 else 0.01\n    for p in parameters:\n        p.data += -lr * p.grad\n\n    if i % 10000 == 0:\n        print(loss.item())\n\nemb = C[Xdev]\nh = torch.tanh(emb.view(-1, c_size*block_size) @ W1 + b1)\nlogits = h @ W2 + b2\nloss = F.cross_entropy(logits, Ydev)\nprint(loss.item())\n\nfor _ in range(20):\n    out = []\n    context = [0] * block_size\n    while True:\n        emb = C[torch.tensor([context])]\n        h = torch.tanh(emb.view(1, c_size*block_size) @ W1 + b1)\n        logits = h @ W2 + b2\n        probs = F.softmax(logits, dim=1)\n        ix = torch.multinomial(probs, num_samples=1, generator=g).item()\n        context = context[1:] + [ix]\n        out.append(ix)\n        if ix ==0 :\n            break\n\n    print(\"\".join(itos[i] for i in out))<\/code><\/pre>\n\n\n\n
\u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
torch.Size([182625, 3]) torch.Size([182625])\ntorch.Size([22655, 3]) torch.Size([22655])\ntorch.Size([22866, 3]) torch.Size([22866])\n45.05998611450195\n4.614674091339111\n2.5040812492370605\n3.028714656829834\n3.449761390686035\n2.4731736183166504\n2.022644281387329\n2.3816988468170166\n2.354505777359009\n1.9231696128845215\n2.955782413482666\n1.9705157279968262\n2.3768727779388428\n1.6785004138946533\n2.2360496520996094\n2.1877331733703613\n2.0761871337890625\n2.1870200634002686\n2.045487403869629\n1.9190195798873901\n2.1788113117218018\nforkelshan.\ncharry.\nkaliquella.\nmilyansh.\nkail.\nalania.\nsaman.\niriousratilee.\nkearonxomitaliq.\nawazem.\nlamrahire.\ngrahira.\nre.\ntrackin.\nkau.\nrubi.\nama.\nleahneud.\ngraesyn.\nges.<\/code><\/pre>\n\n\n\n
\u7ed3\u8bed<\/h2>\n\n\n\n
\u5728\u8fd9\u4e00\u8282\u8bfe\u4e2d\uff0c\u6211\u4eec\u8bad\u7ec3\u7684\u6a21\u578b\u4e0d\u662f\u53ea\u901a\u8fc7\u4e0a\u4e00\u4e2a\u5b57\u7b26\u63a8\u5bfc\u4e0b\u4e00\u4e2a\u5b57\u7b26\uff0c\u800c\u662f\u901a\u8fc7\u524d\u4e09\u4e2a\u5b57\u7b26\u63a8\u5bfc\u540e\u4e00\u4e2a\u5b57\u7b26\uff0c\u5176\u635f\u5931\u7387\u5df2\u7ecf\u964d\u5230\u4e862.45\u4ee5\u4e0b\uff0c\u8fbe\u5230\u4e862.17\u5de6\u53f3\uff0c\u53d6\u5f97\u4e86\u8f83\u5927\u7684\u8fdb\u6b65\uff0c\u5e76\u4e14\u8f93\u51fa\u7684\u4e5f\u66f4\u50cf\u4e00\u4e2a\u6b63\u5e38\u7684\u82f1\u6587\u540d\u5b57\u4e86\u3002\u5f53\u7136\u8fd9\u91cc\u8fd8\u4f18\u5316\u7684\u5730\u65b9\uff0c\u5305\u62ec\u5bf9\u5b66\u4e60\u7387\u8fdb\u884c\u4f18\u5316\uff0c\u5bf9\u5d4c\u5c42\u7684\u7ef4\u5ea6\u8fdb\u884c\u4f18\u5316\uff0c\u5bf9\u9690\u85cf\u5c42\u7684\u795e\u7ecf\u5143\u6570\u91cf\u8fdb\u884c\u4f18\u5316\u7b49\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"
\u8fd9\u662fAndrej Karpathy \u96f6\u57fa\u7840\u5b66\u4e60\u795e\u7ecf\u7f51\u7edc\u7684\u89c6\u9891\u8bfe\u7684\u7b2c\u4e09\u8282\u8bfe\uff0c\u5728\u8fd9\u4e00\u8282\u8bfe\u4e2d\uff0c\u6211\u4eec\u4f1a\u5b9e\u73b0\u4e00\u4e2a\u7b80\u5355\u7684\u82f1 […]<\/p>\n","protected":false},"author":1,"featured_media":2425,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_uag_custom_page_level_css":"","footnotes":""},"categories":[10],"tags":[29,24,28],"class_list":["post-2425","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-10","tag-andrej-karpathy","tag-24","tag-28"],"uagb_featured_image_src":{"full":["https:\/\/pic.zjml.life\/api\/image\/17866118491520bcc48?quality=mid",800,600,false],"thumbnail":["https:\/\/pic.zjml.life\/api\/image\/17866118491520bcc48?quality=mid",150,150,true],"medium":["https:\/\/pic.zjml.life\/api\/image\/17866118491520bcc48?quality=mid",300,300,false],"medium_large":["https:\/\/pic.zjml.life\/api\/image\/17866118491520bcc48?quality=mid",768,0,false],"large":["https:\/\/pic.zjml.life\/api\/image\/17866118491520bcc48?quality=mid",1024,1024,false],"1536x1536":["https:\/\/pic.zjml.life\/api\/image\/17866118491520bcc48?quality=mid",1536,1536,false],"2048x2048":["https:\/\/pic.zjml.life\/api\/image\/17866118491520bcc48?quality=mid",2048,2048,false]},"uagb_author_info":{"display_name":"zjml","author_link":"https:\/\/blog.zjml.life\/?author=1"},"uagb_comment_info":0,"uagb_excerpt":"\u8fd9\u662fAndrej Karpathy \u96f6\u57fa\u7840\u5b66\u4e60\u795e\u7ecf\u7f51\u7edc\u7684\u89c6\u9891\u8bfe\u7684\u7b2c\u4e09\u8282\u8bfe\uff0c\u5728\u8fd9\u4e00\u8282\u8bfe\u4e2d\uff0c\u6211\u4eec\u4f1a\u5b9e\u73b0\u4e00\u4e2a\u7b80\u5355\u7684\u82f1…","_links":{"self":[{"href":"https:\/\/blog.zjml.life\/index.php?rest_route=\/wp\/v2\/posts\/2425","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/blog.zjml.life\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/blog.zjml.life\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/blog.zjml.life\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/blog.zjml.life\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=2425"}],"version-history":[{"count":7,"href":"https:\/\/blog.zjml.life\/index.php?rest_route=\/wp\/v2\/posts\/2425\/revisions"}],"predecessor-version":[{"id":2438,"href":"https:\/\/blog.zjml.life\/index.php?rest_route=\/wp\/v2\/posts\/2425\/revisions\/2438"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/blog.zjml.life\/index.php?rest_route=\/wp\/v2\/posts\/2425"}],"wp:attachment":[{"href":"https:\/\/blog.zjml.life\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=2425"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/blog.zjml.life\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=2425"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/blog.zjml.life\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=2425"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}