{"id":2406,"date":"2026-08-09T18:16:36","date_gmt":"2026-08-09T10:16:36","guid":{"rendered":"https:\/\/blog.zjml.life\/?p=2406"},"modified":"2026-08-09T18:17:39","modified_gmt":"2026-08-09T10:17:39","slug":"%e8%b7%9f%e7%9d%80-andrej-karpathy-%e9%9b%b6%e5%9f%ba%e7%a1%80%e5%ad%a6%e4%b9%a0%e7%a5%9e%e7%bb%8f%e7%bd%91%e7%bb%9c-%e8%af%ad%e8%a8%80%e5%bb%ba%e6%a8%a1%e5%85%a5%e9%97%a8-%e5%ad%a6%e4%b9%a0%e7%ac%94","status":"publish","type":"post","link":"https:\/\/blog.zjml.life\/?p=2406","title":{"rendered":"\u8ddf\u7740 Andrej Karpathy \u96f6\u57fa\u7840\u5b66\u4e60\u795e\u7ecf\u7f51\u7edc \u8bed\u8a00\u5efa\u6a21\u5165\u95e8 \u5b66\u4e60\u7b14\u8bb0"},"content":{"rendered":"\n

\u8fd9\u662fAndrej Karpathy \u96f6\u57fa\u7840\u5b66\u4e60\u795e\u7ecf\u7f51\u7edc\u7684\u89c6\u9891\u8bfe\u7684\u7b2c\u4e8c\u8282\u8bfe\uff0c\u5728\u8fd9\u4e00\u8282\u8bfe\u4e2d\uff0c\u6211\u4eec\u4f1a\u5b9e\u73b0\u4e00\u4e2a\u7b80\u5355\u7684\u82f1\u6587\u540d\u751f\u6210\u7f51\u7edc\u3002<\/p>\n\n\n\n

\u51c6\u5907\u5de5\u4f5c<\/h2><\/div>\n\n\n\n

\u5b89\u88c5Python
\u5b89\u88c5Pytorch
\u89c2\u770b\u89c6\u9891\u8bfe\u7a0b<\/a><\/p>\n\n\n\n

\u6b63\u5f0f\u5f00\u59cb\u7f16\u5199\u4ee3\u7801<\/h2><\/div>\n\n\n\n

\u51c6\u5907\u59d3\u540d\u96c6<\/h3><\/div>\n\n\n\n

\u9996\u5148\uff0c\u6211\u4eec\u9700\u8981\u51c6\u5907\u597d\u59d3\u540d\u6587\u672c\uff0c\u8fd9\u53ef\u4ee5\u4eceGitHub\u4e0a\u76f4\u63a5\u4e0b\u8f7d<\/a>\uff0c\u8bfb\u8005\u53ef\u4ee5\u70b9\u51fb\u76f4\u63a5\u4e0b\u8f7d<\/a>\u8fdb\u884c\u4e0b\u8f7d\u3002\u7b14\u8005\u5df2\u7ecf\u628a\u94fe\u63a5\u8bbe\u7f6e\u597d\u4e86\u3002<\/p>\n\n\n\n

\u6211\u4eec\u5148\u6765\u5199\u4e00\u4e2amain.py\uff0c\u770b\u770b\u6211\u4eec\u7684\u59d3\u540d\u96c6\uff1a<\/p>\n\n\n\n

words = open('names.txt','r').read().splitlines()\nprint(words[:10])\nprint(len(words))\nprint(min(len(word) for word in words))\nprint(max(len(word) for word in words))<\/code><\/pre>\n\n\n\n
\u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
D:\\Data\\Anacondea\\envs\\lora\\python.exe D:\\Data\\Python\\Class_2\\main.py \n['emma', 'olivia', 'ava', 'isabella', 'sophia', 'charlotte', 'mia', 'amelia', 'harper', 'evelyn']\n32033\n2\n15<\/code><\/pre>\n\n\n\n
\u53ef\u4ee5\u770b\u5230\u603b\u5171\u670932033\u4e2a\u59d3\u540d\u96c6\uff0c\u8bf4\u660e\u6211\u4eec\u7684\u59d3\u540d\u96c6\u52a0\u8f7d\u5df2\u7ecf\u6210\u529f\u4e86\u3002<\/p>\n\n\n\n
\u51c6\u5907\u6837\u672c\u96c6<\/h3><\/div>\n\n\n\n
\u7531\u4e8e\u6211\u4eec\u8fd9\u4e2a\u6a21\u578b\u7eaf\u7cb9\u7684\u53ea\u662f\u4f9d\u636e\u4e0a\u4e00\u4e2a\u5b57\u6bcd\u53bb\u731c\u4e0b\u4e00\u4e2a\u5b57\u6bcd\uff0c\u56e0\u6b64\uff0c\u6211\u4eec\u9700\u8981\u7edf\u8ba1\u6574\u4e2a\u59d3\u540d\u96c6\u4e2d\uff0c\u6bcf\u4e00\u4e2a\u4e0a\u4e00\u4e2a\u5b57\u6bcd\u4e0e\u4e0b\u4e00\u4e2a\u5b57\u6bcd\u7684\u7ec4\u5408\u51fa\u73b0\u4e86\u591a\u5c11\u6b21\u3002\u5176\u6b21\u6570\u8d8a\u591a\uff0c\u8bf4\u660e\u8fd9\u6837\u7684\u6982\u7387\u8d8a\u5927\uff0c\u5176\u6b21\u6570\u8d8a\u5c11\uff0c\u8bf4\u660e\u8fd9\u6837\u7684\u6982\u7387\u8d8a\u4f4e\u3002\u9996\u5148\u6211\u4eec\u9700\u8981\u628a\u8fd9\u4e2a\u59d3\u540d\u96c6\u4e2d\u7684\u6bcf\u4e00\u4e2a\u59d3\u540d\u7684\u524d\u9762\u548c\u540e\u9762\u5404\u63d2\u5165\u4e00\u4e2a\u8d77\u59cb\u7b26\u548c\u7ec8\u6b62\u7b26\uff0c\u8fd9\u6837\u6a21\u578b\u624d\u80fd\u77e5\u9053\u5728\u8fd9\u4e2a\u5b57\u6bcd\u4e4b\u540e\uff0c\u6709\u591a\u5927\u7684\u6982\u7387\u76f4\u63a5\u7ec8\u6b62\uff0c\u7ed3\u675f\u8fd9\u4e2a\u540d\u5b57\u7684\u751f\u6210\uff1a<\/p>\n\n\n\n
words = open('names.txt','r').read().splitlines()\nb = {}\nfor word in words:\n    chs = ['<S>'] + list(word) + ['<E>']\n    for i in range(len(chs) -1):\n        bigram = (chs[i], chs[i+1])\n        if bigram in b:\n            b[bigram] += 1\n        else:\n            b[bigram] = 1\n\nprint(sorted(b.items(), key=lambda kv: kv[-1], reverse = True))<\/code><\/pre>\n\n\n\n
\u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
D:\\Data\\Anacondea\\envs\\lora\\python.exe D:\\Data\\Python\\Class_2\\main.py \n[(('n', '<E>'), 6763), (('a', '<E>'), 6640), (('a', 'n'), 5438), (('<S>', 'a'), 4410), (('e', '<E>'), 3983), (('a', 'r'), 3264), (('e', 'l'), 3248), (('r', 'i'), 3033), (('n', 'a'), 2977), (('<S>', 'k'), 2963), (('l', 'e'), 2921), (('e', 'n'), 2675), (('l', 'a'), 2623), (('m', 'a'), 2590), (('<S>', 'm'), 2538), (('a', 'l'), 2528), (('i', '<E>'), 2489), (('l', 'i'), 2480), (('i', 'a'), 2445), (('<S>', 'j'), 2422), (('o', 'n'), 2411), (('h', '<E>'), 2409), (('r', 'a'), 2356), (('a', 'h'), 2332), (('h', 'a'), 2244), (('y', 'a'), 2143), (('i', 'n'), 2126), (('<S>', 's'), 2055), (('a', 'y'), 2050), (('y', '<E>'), 2007), (('e', 'r'), 1958), (('n', 'n'), 1906), (('y', 'n'), 1826), (('k', 'a'), 1731), (('n', 'i'), 1725), (('r', 'e'), 1697), (('<S>', 'd'), 1690), (('i', 'e'), 1653), (('a', 'i'), 1650), (('<S>', 'r'), 1639), (('a', 'm'), 1634), (('l', 'y'), 1588), (('<S>', 'l'), 1572), (('<S>', 'c'), 1542), (('<S>', 'e'), 1531), (('j', 'a'), 1473), (('r', '<E>'), 1377), (('n', 'e'), 1359), (('l', 'l'), 1345), (('i', 'l'), 1345), (('i', 's'), 1316), (('l', '<E>'), 1314), (('<S>', 't'), 1308), (('<S>', 'b'), 1306), (('d', 'a'), 1303), (('s', 'h'), 1285), (('d', 'e'), 1283), (('e', 'e'), 1271), (('m', 'i'), 1256), (('s', 'a'), 1201), (('s', '<E>'), 1169), (('<S>', 'n'), 1146), (('a', 's'), 1118), (('y', 'l'), 1104), (('e', 'y'), 1070), (('o', 'r'), 1059), (('a', 'd'), 1042), (('t', 'a'), 1027), (('<S>', 'z'), 929), (('v', 'i'), 911), (('k', 'e'), 895), (('s', 'e'), 884), (('<S>', 'h'), 874), (('r', 'o'), 869), (('e', 's'), 861), (('z', 'a'), 860), (('o', '<E>'), 855), (('i', 'r'), 849), (('b', 'r'), 842), (('a', 'v'), 834), (('m', 'e'), 818), (('e', 'i'), 818), (('c', 'a'), 815), (('i', 'y'), 779), (('r', 'y'), 773), (('e', 'm'), 769), (('s', 't'), 765), (('h', 'i'), 729), (('t', 'e'), 716), (('n', 'd'), 704), (('l', 'o'), 692), (('a', 'e'), 692), (('a', 't'), 687), (('s', 'i'), 684), (('e', 'a'), 679), (('d', 'i'), 674), (('h', 'e'), 674), (('<S>', 'g'), 669), (('t', 'o'), 667), (('c', 'h'), 664), (('b', 'e'), 655), (('t', 'h'), 647), (('v', 'a'), 642), (('o', 'l'), 619), (('<S>', 'i'), 591), (('i', 'o'), 588), (('e', 't'), 580), (('v', 'e'), 568), (('a', 'k'), 568), (('a', 'a'), 556), (('c', 'e'), 551), (('a', 'b'), 541), (('i', 't'), 541), (('<S>', 'y'), 535), (('t', 'i'), 532), (('s', 'o'), 531), (('m', '<E>'), 516), (('d', '<E>'), 516), (('<S>', 'p'), 515), (('i', 'c'), 509), (('k', 'i'), 509), (('o', 's'), 504), (('n', 'o'), 496), (('t', '<E>'), 483), (('j', 'o'), 479), (('u', 's'), 474), (('a', 'c'), 470), (('n', 'y'), 465), (('e', 'v'), 463), (('s', 's'), 461), (('m', 'o'), 452), (('i', 'k'), 445), (('n', 't'), 443), (('i', 'd'), 440), (('j', 'e'), 440), (('a', 'z'), 435), (('i', 'g'), 428), (('i', 'm'), 427), (('r', 'r'), 425), (('d', 'r'), 424), (('<S>', 'f'), 417), (('u', 'r'), 414), (('r', 'l'), 413), (('y', 's'), 401), (('<S>', 'o'), 394), (('e', 'd'), 384), (('a', 'u'), 381), (('c', 'o'), 380), (('k', 'y'), 379), (('d', 'o'), 378), (('<S>', 'v'), 376), (('t', 't'), 374), (('z', 'e'), 373), (('z', 'i'), 364), (('k', '<E>'), 363), (('g', 'h'), 360), (('t', 'r'), 352), (('k', 'o'), 344), (('t', 'y'), 341), (('g', 'e'), 334), (('g', 'a'), 330), (('l', 'u'), 324), (('b', 'a'), 321), (('d', 'y'), 317), (('c', 'k'), 316), (('<S>', 'w'), 307), (('k', 'h'), 307), (('u', 'l'), 301), (('y', 'e'), 301), (('y', 'r'), 291), (('m', 'y'), 287), (('h', 'o'), 287), (('w', 'a'), 280), (('s', 'l'), 279), (('n', 's'), 278), (('i', 'z'), 277), (('u', 'n'), 275), (('o', 'u'), 275), (('n', 'g'), 273), (('y', 'd'), 272), (('c', 'i'), 271), (('y', 'o'), 271), (('i', 'v'), 269), (('e', 'o'), 269), (('o', 'm'), 261), (('r', 'u'), 252), (('f', 'a'), 242), (('b', 'i'), 217), (('s', 'y'), 215), (('n', 'c'), 213), (('h', 'y'), 213), (('p', 'a'), 209), (('r', 't'), 208), (('q', 'u'), 206), (('p', 'h'), 204), (('h', 'r'), 204), (('j', 'u'), 202), (('g', 'r'), 201), (('p', 'e'), 197), (('n', 'l'), 195), (('y', 'i'), 192), (('g', 'i'), 190), (('o', 'd'), 190), (('r', 's'), 190), (('r', 'd'), 187), (('h', 'l'), 185), (('s', 'u'), 185), (('a', 'x'), 182), (('e', 'z'), 181), (('e', 'k'), 178), (('o', 'v'), 176), (('a', 'j'), 175), (('o', 'h'), 171), (('u', 'e'), 169), (('m', 'm'), 168), (('a', 'g'), 168), (('h', 'u'), 166), (('x', '<E>'), 164), (('u', 'a'), 163), (('r', 'm'), 162), (('a', 'w'), 161), (('f', 'i'), 160), (('z', '<E>'), 160), (('u', '<E>'), 155), (('u', 'm'), 154), (('e', 'c'), 153), (('v', 'o'), 153), (('e', 'h'), 152), (('p', 'r'), 151), (('d', 'd'), 149), (('o', 'a'), 149), (('w', 'e'), 149), (('w', 'i'), 148), (('y', 'm'), 148), (('z', 'y'), 147), (('n', 'z'), 145), (('y', 'u'), 141), (('r', 'n'), 140), (('o', 'b'), 140), (('k', 'l'), 139), (('m', 'u'), 139), (('l', 'd'), 138), (('h', 'n'), 138), (('u', 'd'), 136), (('<S>', 'x'), 134), (('t', 'l'), 134), (('a', 'f'), 134), (('o', 'e'), 132), (('e', 'x'), 132), (('e', 'g'), 125), (('f', 'e'), 123), (('z', 'l'), 123), (('u', 'i'), 121), (('v', 'y'), 121), (('e', 'b'), 121), (('r', 'h'), 121), (('j', 'i'), 119), (('o', 't'), 118), (('d', 'h'), 118), (('h', 'm'), 117), (('c', 'l'), 116), (('o', 'o'), 115), (('y', 'c'), 115), (('o', 'w'), 114), (('o', 'c'), 114), (('f', 'r'), 114), (('b', '<E>'), 114), (('m', 'b'), 112), (('z', 'o'), 110), (('i', 'b'), 110), (('i', 'u'), 109), (('k', 'r'), 109), (('g', '<E>'), 108), (('y', 'v'), 106), (('t', 'z'), 105), (('b', 'o'), 105), (('c', 'y'), 104), (('y', 't'), 104), (('u', 'b'), 103), (('u', 'c'), 103), (('x', 'a'), 103), (('b', 'l'), 103), (('o', 'y'), 103), (('x', 'i'), 102), (('i', 'f'), 101), (('r', 'c'), 99), (('c', '<E>'), 97), (('m', 'r'), 97), (('n', 'u'), 96), (('o', 'p'), 95), (('i', 'h'), 95), (('k', 's'), 95), (('l', 's'), 94), (('u', 'k'), 93), (('<S>', 'q'), 92), (('d', 'u'), 92), (('s', 'm'), 90), (('r', 'k'), 90), (('i', 'x'), 89), (('v', '<E>'), 88), (('y', 'k'), 86), (('u', 'w'), 86), (('g', 'u'), 85), (('b', 'y'), 83), (('e', 'p'), 83), (('g', 'o'), 83), (('s', 'k'), 82), (('u', 't'), 82), (('a', 'p'), 82), (('e', 'f'), 82), (('i', 'i'), 82), (('r', 'v'), 80), (('f', '<E>'), 80), (('t', 'u'), 78), (('y', 'z'), 78), (('<S>', 'u'), 78), (('l', 't'), 77), (('r', 'g'), 76), (('c', 'r'), 76), (('i', 'j'), 76), (('w', 'y'), 73), (('z', 'u'), 73), (('l', 'v'), 72), (('h', 't'), 71), (('j', '<E>'), 71), (('x', 't'), 70), (('o', 'i'), 69), (('e', 'u'), 69), (('o', 'k'), 68), (('b', 'd'), 65), (('a', 'o'), 63), (('p', 'i'), 61), (('s', 'c'), 60), (('d', 'l'), 60), (('l', 'm'), 60), (('a', 'q'), 60), (('f', 'o'), 60), (('p', 'o'), 59), (('n', 'k'), 58), (('w', 'n'), 58), (('u', 'h'), 58), (('e', 'j'), 55), (('n', 'v'), 55), (('s', 'r'), 55), (('o', 'z'), 54), (('i', 'p'), 53), (('l', 'b'), 52), (('i', 'q'), 52), (('w', '<E>'), 51), (('m', 'c'), 51), (('s', 'p'), 51), (('e', 'w'), 50), (('k', 'u'), 50), (('v', 'r'), 48), (('u', 'g'), 47), (('o', 'x'), 45), (('u', 'z'), 45), (('z', 'z'), 45), (('j', 'h'), 45), (('b', 'u'), 45), (('o', 'g'), 44), (('n', 'r'), 44), (('f', 'f'), 44), (('n', 'j'), 44), (('z', 'h'), 43), (('c', 'c'), 42), (('r', 'b'), 41), (('x', 'o'), 41), (('b', 'h'), 41), (('p', 'p'), 39), (('x', 'l'), 39), (('h', 'v'), 39), (('b', 'b'), 38), (('m', 'p'), 38), (('x', 'x'), 38), (('u', 'v'), 37), (('x', 'e'), 36), (('w', 'o'), 36), (('c', 't'), 35), (('z', 'm'), 35), (('t', 's'), 35), (('m', 's'), 35), (('c', 'u'), 35), (('o', 'f'), 34), (('u', 'x'), 34), (('k', 'w'), 34), (('p', '<E>'), 33), (('g', 'l'), 32), (('z', 'r'), 32), (('d', 'n'), 31), (('g', 't'), 31), (('g', 'y'), 31), (('h', 's'), 31), (('x', 's'), 31), (('g', 's'), 30), (('x', 'y'), 30), (('y', 'g'), 30), (('d', 'm'), 30), (('d', 's'), 29), (('h', 'k'), 29), (('y', 'x'), 28), (('q', '<E>'), 28), (('g', 'n'), 27), (('y', 'b'), 27), (('g', 'w'), 26), (('n', 'h'), 26), (('k', 'n'), 26), (('g', 'g'), 25), (('d', 'g'), 25), (('l', 'c'), 25), (('r', 'j'), 25), (('w', 'u'), 25), (('l', 'k'), 24), (('m', 'd'), 24), (('s', 'w'), 24), (('s', 'n'), 24), (('h', 'd'), 24), (('w', 'h'), 23), (('y', 'j'), 23), (('y', 'y'), 23), (('r', 'z'), 23), (('d', 'w'), 23), (('w', 'r'), 22), (('t', 'n'), 22), (('l', 'f'), 22), (('y', 'h'), 22), (('r', 'w'), 21), (('s', 'b'), 21), (('m', 'n'), 20), (('f', 'l'), 20), (('w', 's'), 20), (('k', 'k'), 20), (('h', 'z'), 20), (('g', 'd'), 19), (('l', 'h'), 19), (('n', 'm'), 19), (('x', 'z'), 19), (('u', 'f'), 19), (('f', 't'), 18), (('l', 'r'), 18), (('p', 't'), 17), (('t', 'c'), 17), (('k', 't'), 17), (('d', 'v'), 17), (('u', 'p'), 16), (('p', 'l'), 16), (('l', 'w'), 16), (('p', 's'), 16), (('o', 'j'), 16), (('r', 'q'), 16), (('y', 'p'), 15), (('l', 'p'), 15), (('t', 'v'), 15), (('r', 'p'), 14), (('l', 'n'), 14), (('e', 'q'), 14), (('f', 'y'), 14), (('s', 'v'), 14), (('u', 'j'), 14), (('v', 'l'), 14), (('q', 'a'), 13), (('u', 'y'), 13), (('q', 'i'), 13), (('w', 'l'), 13), (('p', 'y'), 12), (('y', 'f'), 12), (('c', 'q'), 11), (('j', 'r'), 11), (('n', 'w'), 11), (('n', 'f'), 11), (('t', 'w'), 11), (('m', 'z'), 11), (('u', 'o'), 10), (('f', 'u'), 10), (('l', 'z'), 10), (('h', 'w'), 10), (('u', 'q'), 10), (('j', 'y'), 10), (('s', 'z'), 10), (('s', 'd'), 9), (('j', 'l'), 9), (('d', 'j'), 9), (('k', 'm'), 9), (('r', 'f'), 9), (('h', 'j'), 9), (('v', 'n'), 8), (('n', 'b'), 8), (('i', 'w'), 8), (('h', 'b'), 8), (('b', 's'), 8), (('w', 't'), 8), (('w', 'd'), 8), (('v', 'v'), 7), (('v', 'u'), 7), (('j', 's'), 7), (('m', 'j'), 7), (('f', 's'), 6), (('l', 'g'), 6), (('l', 'j'), 6), (('j', 'w'), 6), (('n', 'x'), 6), (('y', 'q'), 6), (('w', 'k'), 6), (('g', 'm'), 6), (('x', 'u'), 5), (('m', 'h'), 5), (('m', 'l'), 5), (('j', 'm'), 5), (('c', 's'), 5), (('j', 'v'), 5), (('n', 'p'), 5), (('d', 'f'), 5), (('x', 'd'), 5), (('z', 'b'), 4), (('f', 'n'), 4), (('x', 'c'), 4), (('m', 't'), 4), (('t', 'm'), 4), (('z', 'n'), 4), (('z', 't'), 4), (('p', 'u'), 4), (('c', 'z'), 4), (('b', 'n'), 4), (('z', 's'), 4), (('f', 'w'), 4), (('d', 't'), 4), (('j', 'd'), 4), (('j', 'c'), 4), (('y', 'w'), 4), (('v', 'k'), 3), (('x', 'w'), 3), (('t', 'j'), 3), (('c', 'j'), 3), (('q', 'w'), 3), (('g', 'b'), 3), (('o', 'q'), 3), (('r', 'x'), 3), (('d', 'c'), 3), (('g', 'j'), 3), (('x', 'f'), 3), (('z', 'w'), 3), (('d', 'k'), 3), (('u', 'u'), 3), (('m', 'v'), 3), (('c', 'x'), 3), (('l', 'q'), 3), (('p', 'b'), 2), (('t', 'g'), 2), (('q', 's'), 2), (('t', 'x'), 2), (('f', 'k'), 2), (('b', 't'), 2), (('j', 'n'), 2), (('k', 'c'), 2), (('z', 'k'), 2), (('s', 'j'), 2), (('s', 'f'), 2), (('z', 'j'), 2), (('n', 'q'), 2), (('f', 'z'), 2), (('h', 'g'), 2), (('w', 'w'), 2), (('k', 'j'), 2), (('j', 'k'), 2), (('w', 'm'), 2), (('z', 'c'), 2), (('z', 'v'), 2), (('w', 'f'), 2), (('q', 'm'), 2), (('k', 'z'), 2), (('j', 'j'), 2), (('z', 'p'), 2), (('j', 't'), 2), (('k', 'b'), 2), (('m', 'w'), 2), (('h', 'f'), 2), (('c', 'g'), 2), (('t', 'f'), 2), (('h', 'c'), 2), (('q', 'o'), 2), (('k', 'd'), 2), (('k', 'v'), 2), (('s', 'g'), 2), (('z', 'd'), 2), (('q', 'r'), 1), (('d', 'z'), 1), (('p', 'j'), 1), (('q', 'l'), 1), (('p', 'f'), 1), (('q', 'e'), 1), (('b', 'c'), 1), (('c', 'd'), 1), (('m', 'f'), 1), (('p', 'n'), 1), (('w', 'b'), 1), (('p', 'c'), 1), (('h', 'p'), 1), (('f', 'h'), 1), (('b', 'j'), 1), (('f', 'g'), 1), (('z', 'g'), 1), (('c', 'p'), 1), (('p', 'k'), 1), (('p', 'm'), 1), (('x', 'n'), 1), (('s', 'q'), 1), (('k', 'f'), 1), (('m', 'k'), 1), (('x', 'h'), 1), (('g', 'f'), 1), (('v', 'b'), 1), (('j', 'p'), 1), (('g', 'z'), 1), (('v', 'd'), 1), (('d', 'b'), 1), (('v', 'h'), 1), (('h', 'h'), 1), (('g', 'v'), 1), (('d', 'q'), 1), (('x', 'b'), 1), (('w', 'z'), 1), (('h', 'q'), 1), (('j', 'b'), 1), (('x', 'm'), 1), (('w', 'g'), 1), (('t', 'b'), 1), (('z', 'x'), 1)]<\/code><\/pre>\n\n\n\n
\u53ef\u4ee5\u770b\u5230\uff0c\u6211\u4eec\u5df2\u7ecf\u5bf9\u6574\u4e2a\u59d3\u540d\u96c6\u505a\u4e86\u4e00\u6b21\u5b8c\u5168\u7684\u7edf\u8ba1\uff0c\u6bcf\u4e00\u79cd\u7ec4\u5408\u90fd\u6709\u5176\u8ba1\u6570\uff08\u5982\u679c\u4e3a0\u5219\u6ca1\u6709\u8ba1\u6570\uff09\u3002\u6211\u4eec\u8089\u773c\u4e0a\u4e5f\u53ef\u4ee5\u89c2\u5bdf\u5230\u4e00\u4e9b\u89c4\u5f8b\uff0c\u6bd4\u5982\u540d\u5b57\u4e2dn\u7ed3\u5c3e\u548ca\u7ed3\u5c3e\u6bd4\u8f83\u591a\uff0c\u8fd9\u4e8b\u5b9e\u4e0a\u5c31\u662f\u6211\u4eec\u4eca\u5929\u6240\u9700\u8981\u4f7f\u7528\u5230\u7684\u6837\u672c\u96c6\u4e86\uff0c\u6a21\u578b\u7eaf\u7cb9\u7684\u5c31\u662f\u5b66\u4e60\u524d\u4e00\u4e2a\u5b57\u7b26\u4ea7\u751f\u540e\u4e00\u4e2a\u5b57\u7b26\u7684\u6982\u7387\u5173\u7cfb\uff0c\u5e76\u4e14\u4ee5\u6b64\u6765\u751f\u6210\u59d3\u540d\u3002<\/p>\n\n\n\n
\u6211\u4eec\u4eca\u5929\u60f3\u4f7f\u7528Pytorch\uff0c\u5148\u751f\u6210\u4e00\u4e2a27\u00d727\u7684\u77e9\u9635\u3002\u56e0\u4e3a\u8d77\u59cb\u7b26\u548c\u7ec8\u6b62\u7b26\u6211\u4eec\u5b9e\u9645\u4e0a\u53ef\u4ee5\u7528\u4e00\u4e2a\u5b57\u7b26\u6765\u4ee3\u66ff\uff0c\u6240\u4ee5\u4e00\u5171\u8fd9\u4e2a\u6a21\u578b\u9700\u8981\u5b66\u4e60\u7684\u5173\u7cfb\u4e2a\u6570\u5c31\u662f26\u4e2a\u5b57\u6bcd\u52a0\u4e0a\u4e00\u4e2a\u8d77\u59cb\u548c\u6216\u7ec8\u6b62\u7b26\u7ec4\u6210\u7684\uff0c\u6240\u4ee5\u4e00\u5171\u662f27\u4e2a\u5b57\u7b26\u3002\u6211\u4eec\u5148\u6253\u5f00\u4e00\u4e2a27\u00d727\u7684\u77e9\u9635\uff0c\u5e76\u4e14\u628a\u8fd927\u4e2a\u5b57\u7b26\u505a\u6210\u4e00\u4e2a\u4e0e\u5176\u4e0b\u6807\u5bf9\u5e94\u7684\u5b57\u5178\uff1a<\/p>\n\n\n\n
words = open('names.txt','r').read().splitlines()\nimport torch\n# \u5efa\u7acb\u4e00\u4e2a27\u00d727\u7684\u7f51\u683c\nN = torch.zeros((27, 27), dtype = torch.int32)\n\n# \u5c06\u59d3\u540d\u4e2d\u51fa\u73b0\u7684\u6240\u6709\u5b57\u6bcd\u505a\u6210\u4e00\u4e2a\u5217\u8868     chars = ['a','b','c','d','e' ...]\nchars = sorted(list(set(''.join(words))))\n# \u6211\u4eec\u5e0c\u671b\u628a\u7b2c0\u4e2a\u4f4d\u7f6e\u8ba9\u7ed9\u5f00\u542f\u6216\u7ed3\u675f\u5360\u4f4d\u7b26'.'       stoi = {'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5 ...}\nstoi = {s:i+1 for i, s in enumerate(chars)}\nstoi['.'] = 0\n# \u5efa\u7acb\u53cd\u5411\u7d22\u5f15        itos = {0: '.', 1: 'a', 2: 'b', 3: 'c', ...}\nitos = {i: s for s, i in stoi.items()}\nprint(stoi)\nprint(itos)<\/code><\/pre>\n\n\n\n
\u6211\u4eec\u8fd0\u884c\u4e00\u4e0b\u770b\u770b\uff1a<\/p>\n\n\n\n
D:\\Data\\Anacondea\\envs\\lora\\python.exe D:\\Data\\Python\\Class_2\\main.py \n{'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5, 'f': 6, 'g': 7, 'h': 8, 'i': 9, 'j': 10, 'k': 11, 'l': 12, 'm': 13, 'n': 14, 'o': 15, 'p': 16, 'q': 17, 'r': 18, 's': 19, 't': 20, 'u': 21, 'v': 22, 'w': 23, 'x': 24, 'y': 25, 'z': 26, '.': 0}\n{1: 'a', 2: 'b', 3: 'c', 4: 'd', 5: 'e', 6: 'f', 7: 'g', 8: 'h', 9: 'i', 10: 'j', 11: 'k', 12: 'l', 13: 'm', 14: 'n', 15: 'o', 16: 'p', 17: 'q', 18: 'r', 19: 's', 20: 't', 21: 'u', 22: 'v', 23: 'w', 24: 'x', 25: 'y', 26: 'z', 0: '.'}\n\n\u8fdb\u7a0b\u5df2\u7ed3\u675f\uff0c\u9000\u51fa\u4ee3\u7801\u4e3a 0\n<\/code><\/pre>\n\n\n\n
\u53ef\u4ee5\u770b\u5230\uff0cstoi\u548citos\u5b57\u5178\u548c\u548c\u6211\u4eec\u9884\u671f\u7684\u6570\u636e\u7ed3\u6784\u4e00\u6837\u3002\u6211\u4eec\u73b0\u5728\u4fee\u6539\u4ee3\u7801\uff0c\u91cd\u65b0\u751f\u6210b\u5b57\u5178\uff0c\u8fd9\u4e00\u6b21\u6211\u4eec\u5c06\u8d77\u59cb\u7b26\u548c\u7ec8\u6b62\u7b26\u90fd\u7528\u201c.\u201d\u6765\u4ee3\u66ff\uff0c\u5e76\u4e14\u628aB\u5b57\u5178\u7684\u6570\u636e\u653e\u5728N\u77e9\u9635\u4e2d\uff1a<\/p>\n\n\n\n
words = open('names.txt','r').read().splitlines()\nimport torch\n# \u5efa\u7acb\u4e00\u4e2a27\u00d727\u7684\u7f51\u683c\nN = torch.zeros((27, 27), dtype = torch.int32)\n\n# \u5c06\u59d3\u540d\u4e2d\u51fa\u73b0\u7684\u6240\u6709\u5b57\u6bcd\u505a\u6210\u4e00\u4e2a\u5217\u8868     chars = ['a','b','c','d','e' ...]\nchars = sorted(list(set(''.join(words))))\n# \u6211\u4eec\u5e0c\u671b\u628a\u7b2c0\u4e2a\u4f4d\u7f6e\u8ba9\u7ed9\u5f00\u542f\u6216\u7ed3\u675f\u5360\u4f4d\u7b26'.'       stoi = {'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5 ...}\nstoi = {s:i+1 for i, s in enumerate(chars)}\nstoi['.'] = 0\n# \u5efa\u7acb\u53cd\u5411\u7d22\u5f15        itos = {0: '.', 1: 'a', 2: 'b', 3: 'c', ...}\nitos = {i: s for s, i in stoi.items()}\n\nb = {}\nfor word in words:\n    chs = ['.'] + list(word) + ['.']\n    for i in range(len(chs) -1):\n        bigram = (chs[i], chs[i+1])\n        if bigram in b:\n            b[bigram] += 1\n        else:\n            b[bigram] = 1\n\nfor (ch1, ch2), count in b.items():\n    ix1 = stoi[ch1]\n    ix2 = stoi[ch2]\n    N[ix1, ix2] = count<\/code><\/pre>\n\n\n\n
\u4e3a\u4e86\u80fd\u591f\u53ef\u89c6\u5316\u663e\u793a\uff0c\u6211\u4eec\u901a\u8fc7matplotlib\u6765\u751f\u6210\u4e00\u4e2a\u56fe\u7247\uff1a<\/p>\n\n\n\n
import matplotlib.pyplot as plt\n\nplt.figure(figsize=(80.0, 80.0))\nplt.imshow(N, cmap='Blues')\nfor i in range(27):\n    for j in range(27):\n        chstr = itos[i] + itos[j]\n        plt.text(j, i, chstr, ha=\"center\", va=\"bottom\", color='gray', fontsize=6)\n        plt.text(j, i, N[i, j].item(), ha=\"center\", va=\"top\", color='gray', fontsize=6)\nplt.axis('off')\nplt.savefig('bigram_heatmap.png', dpi=150, bbox_inches='tight')\nplt.show()<\/code><\/pre>\n\n\n\n
\u6211\u4eec\u6765\u8fd0\u884c\u4e00\u4e0b\uff0c\u770b\u770b\u6548\u679c\uff1a<\/p>\n\n\n\n
\"\"<\/div><\/figure>\n\n\n\n
\u6839\u636e\u7edf\u8ba1\u51fa\u6765\u7684\u6982\u7387\u751f\u6210\u968f\u673a\u82f1\u6587\u540d<\/h3><\/div>\n\n\n\n
\u90a3\u73b0\u5728N[0]<\/code>\u5c31\u662f\u8fd9\u4e00\u4e2a\u9635\u5217\u4e2d\u6bcf\u4e00\u884c\u7684\u6570\u5b57\u3002\u7531\u4e8e\u6211\u4eec\u6700\u7ec8\u662f\u6839\u636e\u4e0a\u4e00\u4e2a\u5b57\u7b26\uff0c\u901a\u8fc7\u6982\u7387\u7edf\u8ba1\u7684\u65b9\u5f0f\u968f\u673a\u51fa\u4e0b\u4e00\u4e2a\u5b57\u7b26\uff0c\u90a3\u6211\u4eec\u9700\u8981\u5c06\u5176\u8fdb\u884c\u5f52\u4e00\u5316\uff1a<\/p>\n\n\n\n
p = N[0].float()\np = p \/ torch.sum(p)\nprint(p)<\/code><\/pre>\n\n\n\n
\u6267\u884c\u4e00\u4e0b\uff0c\u770b\u4e00\u4e0b\u8f93\u51fa\uff1a<\/p>\n\n\n\n
tensor([0.0000, 0.1377, 0.0408, 0.0481, 0.0528, 0.0478, 0.0130, 0.0209, 0.0273,\n        0.0184, 0.0756, 0.0925, 0.0491, 0.0792, 0.0358, 0.0123, 0.0161, 0.0029,\n        0.0512, 0.0642, 0.0408, 0.0024, 0.0117, 0.0096, 0.0042, 0.0167, 0.0290])<\/code><\/pre>\n\n\n\n
\u53ef\u4ee5\u770b\u5230\u73b0\u5728\u8fd9\u4e9b\u6570\u5b57\u5c31\u662f\u7528\u6765\u8868\u793a\u6982\u7387\u4e86\uff0c\u800c\u4e0d\u662f\u50cf\u4e4b\u524d\u90a3\u6837\u8868\u793a\u5176\u51fa\u73b0\u7684\u6b21\u6570\u3002\u73b0\u5728\u6211\u4eec\u5c1d\u8bd5\u4e00\u4e0b\u5bf9\u7b2c0\u884c\u6839\u636e\u6982\u7387\u8f93\u51fa\u4e00\u4e2a\u5b57\u7b26\uff0c\u5176\u610f\u4e49\u4e5f\u5c31\u662f\u6839\u636e\u6982\u7387\u8f93\u51fa\u8fd9\u4e2a\u540d\u5b57\u7684\u7b2c\u4e00\u4e2a\u5b57\u6bcd\uff1a<\/p>\n\n\n\n
g = torch.Generator().manual_seed(2147483647)\nix = torch.multinomial(p, num_samples=1, replacement = True, generator=g).item()\nprint(itos[ix])<\/code><\/pre>\n\n\n\n
\u6211\u4eec\u901a\u8fc7torch.Generator().manual_seed(2147483647)<\/code>\u56fa\u5b9a\u4e86\u968f\u673a\u79cd\u5b50\uff0c\u8fd9\u6837\u6211\u4eec\u672a\u6765\u7684\u6bcf\u4e00\u6b21\u8fd0\u884c\u90fd\u4f1a\u6709\u76f8\u540c\u7684\u8f93\u51fa\u3002multinomial().item()<\/code>\u65b9\u6cd5\u6700\u540e\u4f1a\u62ff\u5230\u4e00\u4e2a\u6570\u5b57\u3002\u4e5f\u5c31\u662fp<\/code>\u4e2d\u7684\u67d0\u4e00\u4e2a\u4e0b\u6807\uff0c\u6211\u4eec\u6267\u884c\u4e00\u4e0b\uff0c\u770b\u770b\u8fd9\u4e00\u6b21\u968f\u673a\u51fa\u4e86\u54ea\u4e00\u4e2a\u5b57\u6bcd\uff1a<\/p>\n\n\n\n
c<\/code><\/pre>\n\n\n\n
\u968f\u673a\u51fa\u4e86C\uff0c\u4e5f\u5c31\u662f\u5176\u51fa\u73b0\u7684\u6982\u7387\u662f4%\uff0c\u4e5f\u7b97\u662f\u62bd\u5230\u5956\u4e86\uff0c\u8fd9\u6982\u7387\u4e5f\u6bd4\u8f83\u4f4e\u5440\u3002<\/p>\n\n\n\n
Pytorch\u63d0\u4f9b\u4e86torch.rand()<\/code>\u65b9\u6cd5\uff0c\u80fd\u591f\u57280\u548c1\u4e4b\u4e2d\u968f\u673a\u751f\u6210\u6307\u5b9a\u6570\u91cf\u7684\u6570\u5b57\uff0c\u5982\u679c\u6211\u4eec\u751f\u6210\u591a\u4e2a\u6570\u5b57\uff0c\u518d\u8fdb\u884c\u5f52\u4e00\u5316\uff0c\u5c31\u53ef\u4ee5\u83b7\u5f97\u4e00\u4e2a\u968f\u673a\u7684\u6982\u7387\u5217\u8868\u3002\u6211\u4eec\u6765\u770b\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
g = torch.Generator().manual_seed(2147483647)\np = torch.rand(3, generator=g)\np = p \/ p.sum()\nprint(p)<\/code><\/pre>\n\n\n\n
\u770b\u4e00\u4e0b\u8f93\u51fa\uff1a<\/p>\n\n\n\n
tensor([0.6064, 0.3033, 0.0903])<\/code><\/pre>\n\n\n\n
\u6211\u4eec\u518d\u6765\u770b\u4e00\u4e0b\uff0c\u4e0d\u540c\u7684\u65b9\u6cd5\u5bf9\u5f20\u91cf\u7ef4\u5ea6\u7684\u53d8\u5316\uff1a<\/p>\n\n\n\n
print(N.shape)\nprint(N.sum(1, keepdim=True).shape)\nprint(N.sum(1).shape)<\/code><\/pre>\n\n\n\n
\u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
torch.Size([27, 27])\ntorch.Size([27, 1])\ntorch.Size([27])<\/code><\/pre>\n\n\n\n
\u53ef\u4ee5\u770b\u5230\uff0c\u539f\u672c\u662f 27 \u884c\u4e58 27 \u5217\u7684\u5411\u91cf\uff0c\u5728\u8fdb\u884c\u4e00\u6b21\u6c42\u548c\u4e4b\u540e\uff0c\u5982\u679c\u6211\u4eec\u6ca1\u6709\u6307\u5b9akeepdim=True\uff0c\u90a3\u4e48PyTorch\u4f1a\u5c06\u5176\u5408\u5e76\u6210\u4e00\u4e2a\u957f\u5ea6\u4e3a 27 \u7684\u4e00\u7ef4\u6570\u7ec4\uff0c\u800c\u5982\u679c\u6211\u4eec\u6307\u5b9akeepdim=True\uff0c\u90a3\u4e48\u5b83\u5c31\u4f1a\u5c06\u5176\u5408\u5e76\u6210\u4e00\u4e2a 27 \u884c 1 \u5217\u7684\u5217\u5411\u91cf\u3002<\/p>\n\n\n\n
\u7531\u4e8e\u5728\u6211\u4eec\u7684\u6837\u672c\u4e2d\uff0c\u53ef\u80fd\u4f1a\u51fa\u73b0\u67d0\u4e00\u79cd\u7ec4\u5408\u7684\u6570\u91cf\u4e3a 0 \u7684\u60c5\u51b5\uff0c\u8fd9\u4f1a\u5bfc\u81f4\u8be5\u60c5\u51b5\u6c38\u8fdc\u4e0d\u53ef\u80fd\u88ab\u968f\u673a\u3002\u56e0\u6b64\uff0c\u6211\u4eec\u53ef\u4ee5\u8fdb\u884c\u5e73\u6ed1\u5904\u7406\uff1a\u5c06\u6bcf\u4e00\u79cd\u53ef\u80fd\u6027\u52a0 1 \u4e4b\u540e\u518d\u8fdb\u884c\u5f52\u4e00\u5316\u3002\u8fd9\u6837\u5c31\u53ef\u4ee5\u786e\u4fdd\u6bcf\u4e00\u79cd\u7ec4\u5408\u81f3\u5c11\u90fd\u6709\u6982\u7387\u51fa\u73b0\uff0c\u800c\u4e0d\u662f\u65ad\u5d16\u5f0f\u5730\u4e3a\u96f6\uff1a<\/p>\n\n\n\n
print(N[0])\nP = (N+1).float()\nP \/= P.sum(1, keepdim=True)\nprint(P[0])<\/code><\/pre>\n\n\n\n
\u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
tensor([   0, 4410, 1306, 1542, 1690, 1531,  417,  669,  874,  591, 2422, 2963,\n        1572, 2538, 1146,  394,  515,   92, 1639, 2055, 1308,   78,  376,  307,\n         134,  535,  929], dtype=torch.int32)\ntensor([3.1192e-05, 1.3759e-01, 4.0767e-02, 4.8129e-02, 5.2745e-02, 4.7785e-02,\n        1.3038e-02, 2.0898e-02, 2.7293e-02, 1.8465e-02, 7.5577e-02, 9.2452e-02,\n        4.9064e-02, 7.9195e-02, 3.5777e-02, 1.2321e-02, 1.6095e-02, 2.9008e-03,\n        5.1154e-02, 6.4130e-02, 4.0830e-02, 2.4641e-03, 1.1759e-02, 9.6070e-03,\n        4.2109e-03, 1.6719e-02, 2.9008e-02])<\/code><\/pre>\n\n\n\n
\u53ef\u4ee5\u770b\u5230\uff0c\u539f\u672c\u5b8c\u5168\u6ca1\u6709\u7684\u60c5\u51b5\u4e5f\u6709\u6982\u7387\u51fa\u73b0\u4e86\uff0c\u53ea\u662f\u6982\u7387\u5f88\u4f4e\u7f62\u4e86\uff0c\u8fd9\u5c31\u662f\u4e00\u79cd\u5e73\u6ed1\u5904\u7406\u3002<\/p>\n\n\n\n
\u73b0\u5728\u6211\u4eec\u5b9e\u9645\u4e0a\u5df2\u7ecf\u53ef\u4ee5\u6839\u636e\u8fd9\u4e2a 27\u00d727 \u7684\u77e9\u9635\uff0c\u5b8c\u5168\u901a\u8fc7\u6982\u7387\u7684\u65b9\u5f0f\u751f\u6210\u540d\u5b57\u4e86\uff0c\u6211\u4eec\u6765\u6d4b\u8bd5\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
g = torch.Generator().manual_seed(2147483647)\nfor i in range(5):\n    out = []\n    ix = 0\n    while True:\n        p = P[ix]\n        ix = torch.multinomial(p, num_samples=1, replacement = True, generator=g).item()\n        out.append(itos[ix])\n        if ix == 0:\n            break\n    print(\"\".join(out))<\/code><\/pre>\n\n\n\n
\u6211\u4eec\u8fd0\u884c\u4e00\u4e0b\uff0c\u8f93\u51fa5\u4e2a\u540d\u5b57\uff1a<\/p>\n\n\n\n
cexze.\nmomasurailezitynn.\nkonimittain.\nllayn.\nka.<\/code><\/pre>\n\n\n\n
\u597d\u5427\uff0c\u8fd9\u4e9b\u770b\u8d77\u6765\u4e0d\u90a3\u4e48\u50cf\u540d\u5b57\uff0c\u4f46\u662f\u81f3\u5c11\u4ece\u539f\u7406\u4e0a\u6211\u4eec\u5b9e\u73b0\u4e86\u3002<\/p>\n\n\n\n
\u8ba1\u7b97\u635f\u5931\u503c<\/h3><\/div>\n\n\n\n
\u73b0\u5728\u6211\u4eec\u518d\u60f3\u4e00\u4e2a\u529e\u6cd5\u6765\u8ba1\u7b97\u5b83\u7684\u635f\u5931\u7387\u3002\u5176\u5b9e\u57fa\u672c\u539f\u7406\u5f88\u7b80\u5355\uff1a\u5bf9\u4e8e\u4e00\u4e2a\u6837\u672c\uff08\u65e2\u53ef\u4ee5\u662f\u6837\u672c\u96c6\u4e2d\u5df2\u6709\u7684\u6837\u672c\uff0c\u4e5f\u53ef\u4ee5\u662f\u6211\u4eec\u521a\u521a\u901a\u8fc7\u6982\u7387\u65b9\u5f0f\u751f\u6210\u7684\u65b0\u7684\u540d\u5b57\u4f5c\u4e3a\u6837\u672c\uff09\uff0c\u90fd\u4f1a\u6709\u82e5\u5e72\u4e2a\u5b57\u6bcd\u5bf9\uff0c\u6bcf\u4e00\u4e2a\u5b57\u6bcd\u5bf9\u4f1a\u6709\u4e00\u4e2a\u51fa\u73b0\u7684\u6982\u7387\u3002\u6211\u4eec\u53ea\u9700\u8981\u5c06\u8fd9\u4e2a\u6982\u7387\u76f8\u4e58\uff0c\u5c31\u662f\u4e00\u4e2a\u5408\u7406\u7684\u635f\u5931\u503c\u4e86\u3002\u4f46\u662f\u7531\u4e8e\u672c\u8eab\u6bcf\u4e00\u4e2a\u5b57\u6bcd\u5bf9\u51fa\u73b0\u7684\u6982\u7387\u90fd\u5f88\u4f4e\uff0c\u5982\u679c\u6211\u4eec\u4e00\u76f4\u8fd9\u6837\u4e58\u4e0b\u53bb\u7684\u8bdd\uff0c\u8fd9\u4e2a\u6570\u5b57\u4f1a\u5f88\u5feb\u5730\u8d8b\u8fd1\u4e8e0\u3002\u56e0\u6b64\uff0c\u6211\u4eec\u8981\u501f\u7528\u5bf9\u6570\u7684\u8fd0\u7b97\u89c4\u5219\uff1alog(a*b*c) = log(a) + log(b) + log(c)\u3002\u6211\u4eec\u5148\u7b97\u51fa\u5b57\u6bcd\u5bf9\u51fa\u73b0\u6982\u7387\u7684\u5bf9\u6570\u503c\uff0c\u7136\u540e\u518d\u8fdb\u884c\u76f8\u52a0\uff0c\u8fd9\u6837\u5c31\u4f1a\u5f97\u5230\u4e00\u4e2a\u5f88\u5927\u7684\u8d1f\u6570\u503c\uff08loge(1)\u4e3a0\uff09\uff0c\u6700\u540e\u6211\u4eec\u518d\u9664\u4ee5\u6837\u672c\u6570\u91cf\uff0c\u5c31\u53ef\u4ee5\u7b97\u51fa\u5f53\u524d\u6837\u672c\u96c6\u7684\u635f\u5931\u503c\u3002<\/p>\n\n\n\n
\u672a\u6765\u6211\u4eec\u8bad\u7ec3\u51fa\u795e\u7ecf\u7f51\u7edc\u6a21\u578b\u4e4b\u540e\uff0c\u7406\u8bba\u4e0a\u5176\u635f\u5931\u503c\u8d8a\u903c\u8fd1\u8fd9\u4e2a\u503c\u8d8a\u597d\uff0c\u4f46\u662f\u6a21\u578b\u7684\u635f\u5931\u503c\u4e0d\u53ef\u80fd\u4f4e\u4e8e\u8fd9\u4e2a\u635f\u5931\u503c\uff0c\u56e0\u4e3a\u6837\u672c\u96c6\u7684\u635f\u5931\u503c\u5c31\u662f\u7406\u8bba\u6700\u4f4e\u635f\u5931\u503c\u4e86\u3002\u6211\u4eec\u5148\u6765\u5b9e\u73b0\u8fd9\u4e2a\u8ba1\u7b97\u635f\u5931\u503c\u7684\u65b9\u6cd5\uff1a<\/p>\n\n\n\n
log_likelihood = 0.0\nn = 0\n\nfor word in words:\n    chs = ['.'] + list(word) + ['.']\n    for ch1, ch2 in zip(chs, chs[1:]):\n       ix1 = stoi[ch1]\n       ix2 = stoi[ch2]\n       prob = P[ix1, ix2]\n       logprob = torch.log(prob)\n       log_likelihood += logprob\n       n += 1\n\nprint(f\"{log_likelihood=}\")\nnll = -log_likelihood\nprint(f\"{nll=}\")\nprint(f\"{nll\/n}\")<\/code><\/pre>\n\n\n\n
\u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
log_likelihood=tensor(-559951.5625)\nnll=tensor(559951.5625)\n2.4543561935424805<\/code><\/pre>\n\n\n\n
\u53ef\u4ee5\u770b\u5230\uff0c\u5176\u672c\u6765\u52a0\u8d77\u6765\u7684\u5bf9\u6570\u503c\u662f-559951.5625\uff0c\u5f53\u6211\u4eec\u9664\u53bb\u6837\u672c\u6570\u91cf\u4e4b\u540e\uff0c\u5f97\u5230\u4e86\u5e73\u5747\u635f\u5931\u503c\u662f2.45\u5de6\u53f3\uff0c\u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u8981\u5c3d\u53ef\u80fd\u5730\u8ba9\u6211\u4eec\u7684\u6a21\u578b\u63a5\u8fd1\u6837\u672c\u7ea7\u7684\u5e73\u5747\u635f\u5931\u503c\u3002<\/p>\n\n\n\n
\u6837\u672c\u96c6\u5411\u91cf\u5316<\/h3><\/div>\n\n\n\n
\u6211\u4eec\u73b0\u5728\u9700\u8981\u51c6\u5907\u7528\u4e8e\u7ed9\u795e\u7ecf\u7f51\u7edc\u8bad\u7ec3\u7684\u6837\u672c\uff0c\u8fd9\u4e2a\u6837\u672c\u96c6\u5b83\u4e0d\u80fd\u518d\u50cf\u4e4b\u524d\u7684\u65b9\u5f0f\u76f4\u63a5\u8f93\u5165\u5b57\u6bcd\uff0c\u56e0\u4e3a\u795e\u7ecf\u7f51\u7edc\u6ca1\u6709\u529e\u6cd5\u76f4\u63a5\u63a5\u6536\u5b57\u6bcd\uff0c\u5728\u8fd9\u6b21\u5b9e\u9a8c\u4e2d\uff0c\u4e00\u5171\u53ea\u670926\u4e2a\u5b57\u6bcd\uff0c\u52a0\u4e0a\u4e00\u4e2a\u7531\u53e5\u53f7\u4ee3\u8868\u7684\u8d77\u59cb\u548c\u7ec8\u6b62\u7b26\uff0c\u90a3\u4e48\u4e00\u5171\u662f27\u79cd\u53ef\u80fd\u6027\u3002\u6211\u4eec\u901a\u8fc7\u4e0a\u4e00\u4e2a\u5b57\u7b26\u63a8\u65ad\u4e0b\u4e00\u4e2a\u5b57\u7b26\uff0c\u5b57\u7b26\u53ea\u670927\u79cd\u53ef\u80fd\u3002\u90a3\u6211\u4eec\u5148\u53ef\u4ee5\u5c06\u7b2c\u4e00\u4e2a\u5b57\u7b26\u6620\u5c04\u4e3a\u5176\u6570\u5b57\uff0c\u518d\u5c06\u5176\u8f6c\u5316\u4e3a\u4e00\u4e2a27\u7ef4\u7684\u5411\u91cf\uff0c\u9664\u4e86\u5b83\u6240\u5728\u7684\u90a3\u4e2a\u4f4d\u7f6e\u7684\u503c\u4e3a1\uff0c\u5176\u4f59\u7684\u4f4d\u7f6e\u90fd\u4e3a\u8fd9\u4e00\u4e2a\u5b57\u7b26\u3002\u540c\u7406\uff0c\u6bcf\u4e00\u4e2a\u5b57\u7b26\u90fd\u53ef\u4ee5\u901a\u8fc7\u8fd9\u4e2a27\u4f4d\u7684\u5411\u91cf\u6765\u8868\u793a\uff0c\u6211\u4eec\u5c06main.py\u6e05\u7a7a\uff0c\u901a\u8fc7\u4ee3\u7801\u6765\u5b9e\u73b0\uff1a<\/p>\n\n\n\n
words = open('names.txt','r').read().splitlines()\nimport torch\n# \u5efa\u7acb\u4e00\u4e2a27\u00d727\u7684\u7f51\u683c\nN = torch.zeros((27, 27), dtype = torch.int32)\n\n# \u5c06\u59d3\u540d\u4e2d\u51fa\u73b0\u7684\u6240\u6709\u5b57\u6bcd\u505a\u6210\u4e00\u4e2a\u5217\u8868     chars = ['a','b','c','d','e' ...]\nchars = sorted(list(set(''.join(words))))\n# \u6211\u4eec\u5e0c\u671b\u628a\u7b2c0\u4e2a\u4f4d\u7f6e\u8ba9\u7ed9\u5f00\u542f\u6216\u7ed3\u675f\u5360\u4f4d\u7b26'.'       stoi = {'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5 ...}\nstoi = {s:i+1 for i, s in enumerate(chars)}\nstoi['.'] = 0\n# \u5efa\u7acb\u53cd\u5411\u7d22\u5f15        itos = {0: '.', 1: 'a', 2: 'b', 3: 'c', ...}\nitos = {i: s for s, i in stoi.items()}\n\nxs, ys = [], []\nfor word in words[:1]:\n    chs = ['.'] + list(word) + ['.']\n    for ch1, ch2 in zip(chs, chs[1:]):\n        ix1 = stoi[ch1]\n        ix2 = stoi[ch2]\n        # print(ch1, ch2, ix1, ix2)\n        xs.append(ix1)\n        ys.append(ix2)\n\nxs = torch.tensor(xs)\nys = torch.tensor(ys)\n\nprint(xs, ys)\n\nimport torch.nn.functional as F\nxenc = F.one_hot(xs, num_classes=27).float()\nprint(xenc)\n\nprint(xenc.dtype)<\/code><\/pre>\n\n\n\n
\u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
tensor([ 0,  5, 13, 13,  1]) tensor([ 5, 13, 13,  1,  0])\ntensor([[1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n         0., 0., 0., 0., 0., 0., 0., 0., 0.],\n        [0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n         0., 0., 0., 0., 0., 0., 0., 0., 0.],\n        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0.,\n         0., 0., 0., 0., 0., 0., 0., 0., 0.],\n        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0.,\n         0., 0., 0., 0., 0., 0., 0., 0., 0.],\n        [0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n         0., 0., 0., 0., 0., 0., 0., 0., 0.]])\ntorch.float32<\/code><\/pre>\n\n\n\n
\u53ef\u4ee5\u770b\u5230\uff0c\u8fd9\u91cc\u6211\u4eec\u53d6\u4e86\u6837\u672c\u96c6\u4e2d\u7684\u6700\u540e\u4e00\u4e2a\u540d\u5b57\u4f5c\u4e3a\u6d4b\u8bd5\u3002F.one_hot()<\/code>\u65b9\u6cd5\u5c31\u662f\u6211\u4eec\u7528\u4e8e\u5c06\u4e00\u4e2a\u6570\u5b57\u6253\u4e3a\u4e00\u4e2a\u5411\u91cf\u7684\u65b9\u6cd5\u3002\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528matplotlib\u6765\u53ef\u89c6\u5316\uff1a<\/p>\n\n\n\n
import matplotlib.pyplot as plt\nplt.imshow(xenc)\nplt.show()<\/code><\/pre>\n\n\n\n
matplotlib\u53ef\u4ee5\u53ef\u89c6\u5316\u6211\u4eec\u7684xenc\uff0c\u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
\"\"<\/div><\/figure>\n\n\n\n
\u5bf9\u6a21\u578b\u8ba1\u7b97\u635f\u5931\u503c<\/h3><\/div>\n\n\n\n
\u6211\u4eec\u521a\u521a\u5176\u5b9e\u5b9e\u73b0\u4e86\u4e00\u4e2a\u8ba1\u7b97\u635f\u5931\u503c\u7684\u65b9\u6cd5\uff0c\u5e76\u4e14\u901a\u8fc7\u90a3\u4e2a\u65b9\u6cd5\u8ba1\u7b97\u4e86\u6574\u4e2a\u6837\u54c1\u96c6\u7684\u635f\u5931\u503c\uff0c\u4f46\u662f\u6211\u4eec\u73b0\u5728\u5e0c\u671b\u8ba1\u7b97\u5f53\u524d\u6a21\u578b\u7684\u635f\u5931\u503c\uff0c\u5982\u4f55\u53bb\u5b9e\u73b0\u5462\uff1f<\/p>\n\n\n\n
import torch.nn.functional as F\ng = torch.Generator().manual_seed(2147483647)\nW = torch.randn((27, 27), generator=g)<\/code><\/pre>\n\n\n\n
\u6211\u4eec\u5148\u5b9a\u4e49\u4e00\u4e2a27\u00d727\u7684\u53c2\u6570\u77e9\u9635\uff0c\u7136\u540e\u7528xenc @ W\u5f97\u5230\u6bcf\u4e00\u4e2a\u8f93\u5165\u7684\u7b2c\u4e00\u4e2a\u5b57\u7b26\u4e4b\u540e\uff0c\u8f93\u51fa27\u4e2a\u5b57\u7b26\u7684\u5404\u81ea\u7684\u201c\u503e\u5411\u5ea6\u201d\uff0c\u7136\u540e\u5bf9\u5176\u53d6\u81ea\u7136\u5bf9\u6570\uff0c\u518d\u5f52\u4e00\u5316\uff0c\u5f97\u5230probs\uff0c\u8fd9\u4e2aprobs\u662f\u4e00\u4e2a\u5217\u8868\uff0c\u8bb0\u5f55\u4e86\u6bcf\u4e00\u4e2a\u8f93\u5165\u5b57\u7b26\u540e\uff0c\u8f93\u51fa27\u4e2a\u5b57\u7b26\u7684\u5404\u81ea\u7684\u6982\u7387\uff0c\u8ba9\u6211\u4eec\u6765\u5b9e\u73b0\u4ee3\u7801\uff1a<\/p>\n\n\n\n
xenc = F.one_hot(xs, num_classes=27).float()\n\nlogits = xenc @ W\nprint(logits)\n# \u53d6\u81ea\u7136\u6307\u6570\ncounts = logits.exp()\nprint(counts)\nprobs = counts \/ counts.sum(1, keepdim=True)\nprint(probs)<\/code><\/pre>\n\n\n\n
\u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
tensor([[ 1.5674, -0.2373, -0.0274,  ..., -0.0707,  2.4968,  2.4448],\n        [ 0.4724,  1.4830,  0.3175,  ..., -0.4275, -2.1259,  0.9604],\n        [ 0.1936,  1.0532,  0.6339,  ...,  1.5447,  0.6006, -0.7091],\n        ...,\n        [ 0.7414, -0.5879, -0.4651,  ..., -0.1388,  1.3096, -0.2580],\n        [ 1.0669,  0.2136, -0.7660,  ...,  1.3405, -0.2175,  0.8627],\n        [ 0.2163, -0.7873, -0.3301,  ..., -0.2630, -0.7552,  0.8191]])\ntensor([[ 4.7940,  0.7888,  0.9730,  ...,  0.9317, 12.1434, 11.5281],\n        [ 1.6038,  4.4060,  1.3737,  ...,  0.6521,  0.1193,  2.6128],\n        [ 1.2136,  2.8669,  1.8850,  ...,  4.6866,  1.8232,  0.4921],\n        ...,\n        [ 2.0989,  0.5555,  0.6281,  ...,  0.8704,  3.7049,  0.7726],\n        [ 2.9062,  1.2382,  0.4649,  ...,  3.8208,  0.8046,  2.3696],\n        [ 1.2415,  0.4551,  0.7189,  ...,  0.7687,  0.4699,  2.2685]])\ntensor([[0.0607, 0.0100, 0.0123,  ..., 0.0118, 0.1537, 0.1459],\n        [0.0290, 0.0796, 0.0248,  ..., 0.0118, 0.0022, 0.0472],\n        [0.0312, 0.0737, 0.0484,  ..., 0.1204, 0.0469, 0.0126],\n        ...,\n        [0.0301, 0.0080, 0.0090,  ..., 0.0125, 0.0531, 0.0111],\n        [0.0634, 0.0270, 0.0101,  ..., 0.0833, 0.0175, 0.0517],\n        [0.0308, 0.0113, 0.0178,  ..., 0.0190, 0.0116, 0.0562]])<\/code><\/pre>\n\n\n\n
\u53ef\u4ee5\u770b\u5230\uff0c\u6700\u540e\u90fd\u662f\u4ee5\u6982\u7387\u7684\u5f62\u5f0f\u51fa\u73b0\u4e86\uff0c\u4e4b\u540e\u6211\u4eec\u518d\u8ba1\u7b97\u8f93\u5165\u7684\u635f\u5931\uff1a<\/p>\n\n\n\n
nlls = torch.zeros(5)\nprint(nlls)\nfor i in range(5):\n    x = xs[i].item()\n    y = ys[i].item()\n    print('--------')\n    print(f'\u4e8c\u5143\u7ec4\u793a\u4f8b {i + 1}: {itos[x]}{itos[y]} (\u7d22\u5f15 {x},{y})')\n    print('\u795e\u7ecf\u7f51\u7edc\u7684\u8f93\u5165:', x)\n    print('\u795e\u7ecf\u7f51\u7edc\u8f93\u51fa\u7684\u6982\u7387\u5206\u5e03:', probs[i])\n    print('\u771f\u5b9e\u7684\u4e0b\u4e00\u4e2a\u5b57\u7b26 (\u6807\u7b7e):', y)\n    p = probs[i,y]\n    print('\u7f51\u7edc\u8d4b\u4e88\u6b63\u786e\u5b57\u7b26\u7684\u6982\u7387:', p.item())\n    logp = torch.log(p)\n    print('\u5bf9\u6570\u4f3c\u7136:', logp.item())\n    nll = -logp\n    print('\u8d1f\u5bf9\u6570\u4f3c\u7136 (\u635f\u5931):', nll.item())\n    nlls[i] = nll\n\nprint('=========')\nprint('\u5e73\u5747\u8d1f\u5bf9\u6570\u4f3c\u7136\uff0c\u5373\u635f\u5931 =', nlls.mean().item())<\/code><\/pre>\n\n\n\n
\u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
--------\n\u4e8c\u5143\u7ec4\u793a\u4f8b 1: .e (\u7d22\u5f15 0,5)\n\u795e\u7ecf\u7f51\u7edc\u7684\u8f93\u5165: 0\n\u795e\u7ecf\u7f51\u7edc\u8f93\u51fa\u7684\u6982\u7387\u5206\u5e03: tensor([0.0607, 0.0100, 0.0123, 0.0042, 0.0168, 0.0123, 0.0027, 0.0232, 0.0137,\n        0.0313, 0.0079, 0.0278, 0.0091, 0.0082, 0.0500, 0.2378, 0.0603, 0.0025,\n        0.0249, 0.0055, 0.0339, 0.0109, 0.0029, 0.0198, 0.0118, 0.1537, 0.1459])\n\u771f\u5b9e\u7684\u4e0b\u4e00\u4e2a\u5b57\u7b26 (\u6807\u7b7e): 5\n\u7f51\u7edc\u8d4b\u4e88\u6b63\u786e\u5b57\u7b26\u7684\u6982\u7387: 0.01228625513613224\n\u5bf9\u6570\u4f3c\u7136: -4.399273872375488\n\u8d1f\u5bf9\u6570\u4f3c\u7136 (\u635f\u5931): 4.399273872375488\n--------\n\u4e8c\u5143\u7ec4\u793a\u4f8b 2: em (\u7d22\u5f15 5,13)\n\u795e\u7ecf\u7f51\u7edc\u7684\u8f93\u5165: 5\n\u795e\u7ecf\u7f51\u7edc\u8f93\u51fa\u7684\u6982\u7387\u5206\u5e03: tensor([0.0290, 0.0796, 0.0248, 0.0521, 0.1989, 0.0289, 0.0094, 0.0335, 0.0097,\n        0.0301, 0.0702, 0.0228, 0.0115, 0.0181, 0.0108, 0.0315, 0.0291, 0.0045,\n        0.0916, 0.0215, 0.0486, 0.0300, 0.0501, 0.0027, 0.0118, 0.0022, 0.0472])\n\u771f\u5b9e\u7684\u4e0b\u4e00\u4e2a\u5b57\u7b26 (\u6807\u7b7e): 13\n\u7f51\u7edc\u8d4b\u4e88\u6b63\u786e\u5b57\u7b26\u7684\u6982\u7387: 0.018050700426101685\n\u5bf9\u6570\u4f3c\u7136: -4.014570713043213\n\u8d1f\u5bf9\u6570\u4f3c\u7136 (\u635f\u5931): 4.014570713043213\n--------\n\u4e8c\u5143\u7ec4\u793a\u4f8b 3: mm (\u7d22\u5f15 13,13)\n\u795e\u7ecf\u7f51\u7edc\u7684\u8f93\u5165: 13\n\u795e\u7ecf\u7f51\u7edc\u8f93\u51fa\u7684\u6982\u7387\u5206\u5e03: tensor([0.0312, 0.0737, 0.0484, 0.0333, 0.0674, 0.0200, 0.0263, 0.0249, 0.1226,\n        0.0164, 0.0075, 0.0789, 0.0131, 0.0267, 0.0147, 0.0112, 0.0585, 0.0121,\n        0.0650, 0.0058, 0.0208, 0.0078, 0.0133, 0.0203, 0.1204, 0.0469, 0.0126])\n\u771f\u5b9e\u7684\u4e0b\u4e00\u4e2a\u5b57\u7b26 (\u6807\u7b7e): 13\n\u7f51\u7edc\u8d4b\u4e88\u6b63\u786e\u5b57\u7b26\u7684\u6982\u7387: 0.026691533625125885\n\u5bf9\u6570\u4f3c\u7136: -3.623408794403076\n\u8d1f\u5bf9\u6570\u4f3c\u7136 (\u635f\u5931): 3.623408794403076\n--------\n\u4e8c\u5143\u7ec4\u793a\u4f8b 4: ma (\u7d22\u5f15 13,1)\n\u795e\u7ecf\u7f51\u7edc\u7684\u8f93\u5165: 13\n\u795e\u7ecf\u7f51\u7edc\u8f93\u51fa\u7684\u6982\u7387\u5206\u5e03: tensor([0.0312, 0.0737, 0.0484, 0.0333, 0.0674, 0.0200, 0.0263, 0.0249, 0.1226,\n        0.0164, 0.0075, 0.0789, 0.0131, 0.0267, 0.0147, 0.0112, 0.0585, 0.0121,\n        0.0650, 0.0058, 0.0208, 0.0078, 0.0133, 0.0203, 0.1204, 0.0469, 0.0126])\n\u771f\u5b9e\u7684\u4e0b\u4e00\u4e2a\u5b57\u7b26 (\u6807\u7b7e): 1\n\u7f51\u7edc\u8d4b\u4e88\u6b63\u786e\u5b57\u7b26\u7684\u6982\u7387: 0.07367686182260513\n\u5bf9\u6570\u4f3c\u7136: -2.6080665588378906\n\u8d1f\u5bf9\u6570\u4f3c\u7136 (\u635f\u5931): 2.6080665588378906\n--------\n\u4e8c\u5143\u7ec4\u793a\u4f8b 5: a. (\u7d22\u5f15 1,0)\n\u795e\u7ecf\u7f51\u7edc\u7684\u8f93\u5165: 1\n\u795e\u7ecf\u7f51\u7edc\u8f93\u51fa\u7684\u6982\u7387\u5206\u5e03: tensor([0.0150, 0.0086, 0.0396, 0.0100, 0.0606, 0.0308, 0.1084, 0.0131, 0.0125,\n        0.0048, 0.1024, 0.0086, 0.0988, 0.0112, 0.0232, 0.0207, 0.0408, 0.0078,\n        0.0899, 0.0531, 0.0463, 0.0309, 0.0051, 0.0329, 0.0654, 0.0503, 0.0091])\n\u771f\u5b9e\u7684\u4e0b\u4e00\u4e2a\u5b57\u7b26 (\u6807\u7b7e): 0\n\u7f51\u7edc\u8d4b\u4e88\u6b63\u786e\u5b57\u7b26\u7684\u6982\u7387: 0.014977526850998402\n\u5bf9\u6570\u4f3c\u7136: -4.201204299926758\n\u8d1f\u5bf9\u6570\u4f3c\u7136 (\u635f\u5931): 4.201204299926758\n=========\n\u5e73\u5747\u8d1f\u5bf9\u6570\u4f3c\u7136\uff0c\u5373\u635f\u5931 = 3.7693049907684326<\/code><\/pre>\n\n\n\n
\u53ef\u4ee5\u770b\u5230\uff0c\u5bf9\u4e8e\u524d5\u4e2a\u6837\u672c\uff0c\u6211\u4eec\u8fd9\u4e2a\u5b8c\u5168\u968f\u673a\u7684\u53c2\u6570\u8868\uff0c\u521d\u59cb\u72b6\u6001\u4e0b\uff0c\u635f\u5931\u503c\u662f3.76\uff0c\u63a5\u4e0b\u6765\u6211\u4eec\u5e94\u8be5\u5bf9\u5176\u8fdb\u884c\u5fae\u8c03\uff0c\u8ba9\u5176\u66f4\u52a0\u9760\u8fd1\u8fd9\u4e2a\u6837\u672c\u96c6\u7684\u635f\u5931\u503c\uff0c\u4e5f\u5c31\u662f2.45\u5de6\u53f3<\/p>\n\n\n\n
\u5bf9\u53c2\u6570\u8fdb\u884c\u53cd\u5411\u4f20\u64ad<\/h3><\/div>\n\n\n\n
\u56e0\u4e3a\u6211\u4eec\u5e0c\u671b\u80fd\u591f\u5bf9W\u8fdb\u884c\u53cd\u5411\u4f20\u64ad\u3002\u56e0\u6b64\uff0c\u6211\u4eec\u9700\u8981\u8bbe\u7f6eW\u53c2\u6570requires_grad=True\u3002\u6211\u4eec\u5148\u4f18\u5316\u4e00\u4e0b\u521a\u624d\u7684\u4ee3\u7801\uff0c\u901a\u8fc7\u77e9\u9635\u548c\u5411\u91cf\u7684\u65b9\u5f0f\u4e00\u6b21\u6027\u7b97\u51fa\u524d5\u4e2a\u6837\u672c\u7684\u635f\u5931\u503c\uff1a<\/p>\n\n\n\n
words = open('names.txt','r').read().splitlines()\nimport torch\n# \u5efa\u7acb\u4e00\u4e2a27\u00d727\u7684\u7f51\u683c\nN = torch.zeros((27, 27), dtype = torch.int32)\n\n# \u5c06\u59d3\u540d\u4e2d\u51fa\u73b0\u7684\u6240\u6709\u5b57\u6bcd\u505a\u6210\u4e00\u4e2a\u5217\u8868     chars = ['a','b','c','d','e' ...]\nchars = sorted(list(set(''.join(words))))\n# \u6211\u4eec\u5e0c\u671b\u628a\u7b2c0\u4e2a\u4f4d\u7f6e\u8ba9\u7ed9\u5f00\u542f\u6216\u7ed3\u675f\u5360\u4f4d\u7b26'.'       stoi = {'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5 ...}\nstoi = {s:i+1 for i, s in enumerate(chars)}\nstoi['.'] = 0\n# \u5efa\u7acb\u53cd\u5411\u7d22\u5f15        itos = {0: '.', 1: 'a', 2: 'b', 3: 'c', ...}\nitos = {i: s for s, i in stoi.items()}\n\nxs, ys = [], []\nfor word in words:\n    chs = ['.'] + list(word) + ['.']\n    for ch1, ch2 in zip(chs, chs[1:]):\n        ix1 = stoi[ch1]\n        ix2 = stoi[ch2]\n        # print(ch1, ch2, ix1, ix2)\n        xs.append(ix1)\n        ys.append(ix2)\n\nxs = torch.tensor(xs)\nys = torch.tensor(ys)\n\nimport torch.nn.functional as F\ng = torch.Generator().manual_seed(2147483647)\nW = torch.randn((27, 27), generator=g, requires_grad=True)\nxenc = F.one_hot(xs, num_classes=27).float()\n\nlogits = xenc @ W\ncounts = logits.exp()\nprobs = counts \/ counts.sum(1, keepdim=True)\n\nrow_indices = torch.arange(5)\nprint(row_indices)\ncorrent_probs = probs[row_indices, ys[:5]]\nprint(corrent_probs)\nlog_corrent_probs = torch.log(corrent_probs)\nprint(log_corrent_probs)\nmean_log_probs = log_corrent_probs.mean()\nprint(mean_log_probs)\nloss = -mean_log_probs\nprint(loss.item())<\/code><\/pre>\n\n\n\n
\u4e4b\u540e\u6211\u4eec\u518d\u8ba9loss\u8fdb\u884c\u53cd\u5411\u4f20\u64ad\uff0c\u8ba1\u7b97\u51faW\u4e2d\u6bcf\u4e00\u4e2a\u53c2\u6570\u7684\u68af\u5ea6\uff0c\u5e76\u4e14\u901a\u8fc7\u5fae\u8c03\u53c2\u6570\u7684\u65b9\u5f0f\u6765\u8bad\u7ec3\uff0c\u6211\u4eec\u5148\u8bad\u7ec3\u4e00\u6b21\uff0c\u770b\u770b\u635f\u5931\u6709\u6ca1\u6709\u4e0b\u964d\uff1a<\/p>\n\n\n\n
W.grad = None\nloss.backward()\nW.data += -0.1 * W.grad\n\nlogits = xenc @ W\ncounts = logits.exp()\nprobs = counts \/ counts.sum(1, keepdim=True)\nrow_indices = torch.arange(5)\nprint(row_indices)\ncorrent_probs = probs[row_indices, ys[:5]]\nprint(corrent_probs)\nlog_corrent_probs = torch.log(corrent_probs)\nprint(log_corrent_probs)\nmean_log_probs = log_corrent_probs.mean()\nprint(mean_log_probs)\nloss = -mean_log_probs\nprint(loss.item())<\/code><\/pre>\n\n\n\n
\u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
tensor([0, 1, 2, 3, 4])\ntensor([0.0123, 0.0181, 0.0267, 0.0737, 0.0150], grad_fn=<IndexBackward0>)\ntensor([-4.3993, -4.0146, -3.6234, -2.6081, -4.2012], grad_fn=<LogBackward0>)\ntensor(-3.7693, grad_fn=<MeanBackward0>)\n3.7693049907684326\ntensor([0, 1, 2, 3, 4])\ntensor([0.0126, 0.0184, 0.0272, 0.0750, 0.0153], grad_fn=<IndexBackward0>)\ntensor([-4.3774, -3.9937, -3.6039, -2.5905, -4.1805], grad_fn=<LogBackward0>)\ntensor(-3.7492, grad_fn=<MeanBackward0>)\n3.7492127418518066<\/code><\/pre>\n\n\n\n
\u53ef\u4ee5\u770b\u5230\uff0c\u7ecf\u8fc7\u4e00\u6b21\u5fae\u8c03\uff0c\u635f\u5931\u503c\u4ece3.76\u53d8\u6210\u4e863.74\uff0c\u8bf4\u660e\u6211\u4eec\u8fd9\u4e2a\u6b65\u9aa4\u662f\u6709\u6548\u7684\u3002\u63a5\u4e0b\u6765\u6211\u4eec\u518d\u5bf9\u8fd9\u4e2a\u8bad\u7ec3\u6b65\u9aa4\u8fdb\u884c\u4f18\u5316\uff0c\u5e76\u4e14\u5faa\u73af\u53cd\u5411\u4f20\u64ad5\u6b21\uff1a<\/p>\n\n\n\n
words = open('names.txt','r').read().splitlines()\nimport torch\n# \u5efa\u7acb\u4e00\u4e2a27\u00d727\u7684\u7f51\u683c\nN = torch.zeros((27, 27), dtype = torch.int32)\n\n# \u5c06\u59d3\u540d\u4e2d\u51fa\u73b0\u7684\u6240\u6709\u5b57\u6bcd\u505a\u6210\u4e00\u4e2a\u5217\u8868     chars = ['a','b','c','d','e' ...]\nchars = sorted(list(set(''.join(words))))\n# \u6211\u4eec\u5e0c\u671b\u628a\u7b2c0\u4e2a\u4f4d\u7f6e\u8ba9\u7ed9\u5f00\u542f\u6216\u7ed3\u675f\u5360\u4f4d\u7b26'.'       stoi = {'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5 ...}\nstoi = {s:i+1 for i, s in enumerate(chars)}\nstoi['.'] = 0\n# \u5efa\u7acb\u53cd\u5411\u7d22\u5f15        itos = {0: '.', 1: 'a', 2: 'b', 3: 'c', ...}\nitos = {i: s for s, i in stoi.items()}\n\nxs, ys = [], []\nfor word in words:\n    chs = ['.'] + list(word) + ['.']\n    for ch1, ch2 in zip(chs, chs[1:]):\n        ix1 = stoi[ch1]\n        ix2 = stoi[ch2]\n        # print(ch1, ch2, ix1, ix2)\n        xs.append(ix1)\n        ys.append(ix2)\n\nxs = torch.tensor(xs)\nys = torch.tensor(ys)\nnum = xs.nelement()\nprint(f\"\u6837\u672c\u6570\u91cf\uff1a{num}\")\n\nimport torch.nn.functional as F\ng = torch.Generator().manual_seed(2147483647)\nW = torch.randn((27, 27), generator=g, requires_grad=True)\nxenc = F.one_hot(xs, num_classes=27).float()\n\nfor i in range(5):\n    logits = xenc @ W\n    counts = logits.exp()\n    probs = counts \/ counts.sum(1, keepdim=True)\n    loss = -probs[torch.arange(num), ys].log().mean()\n    print(loss.item())\n\n    W.grad = None\n    loss.backward()\n    W.data += -0.1 * W.grad\n\nlogits = xenc @ W\ncounts = logits.exp()\nprobs = counts \/ counts.sum(1, keepdim=True)\nloss = -probs[torch.arange(num), ys].log().mean()\nprint(loss.item())<\/code><\/pre>\n\n\n\n
\u8fd0\u884c\u4e00\u4e0b\uff1a<\/p>\n\n\n\n
\u6837\u672c\u6570\u91cf\uff1a228146\n3.758953809738159\n3.758040428161621\n3.7571277618408203\n3.756216049194336\n3.7553062438964844\n3.754396915435791<\/code><\/pre>\n\n\n\n
\u53ef\u4ee5\u770b\u5230\u635f\u5931\u503c\u6b63\u5728\u7a33\u5b9a\u5730\u4e0b\u964d\u3002<\/p>\n\n\n\n
\u8ba9\u6a21\u578b\u8f93\u51fa<\/h3><\/div>\n\n\n\n
\u6211\u4eec\u5f53\u7136\u8fd8\u9700\u8981\u4e00\u4e2a\u65b9\u6cd5\uff0c\u8ba9\u5fae\u8c03\u4e4b\u540e\u7684\u6a21\u578b\u8f93\u51fa\u540d\u5b57\uff0c\u5e76\u4e14\u8ba9\u53cd\u5411\u4f20\u64ad\u7684\u6b21\u6570\u589e\u52a0\u523050\u6b21\uff0c\u8ba9\u8fd9\u4e2a\u6a21\u578b\u7684\u635f\u5931\u503c\u5c3d\u53ef\u80fd\u7684\u4f4e\uff0c\u4ee5\u4e0b\u662f\u5b8c\u6574\u4ee3\u7801\uff1a<\/p>\n\n\n\n
words = open('names.txt','r').read().splitlines()\nimport torch\n# \u5efa\u7acb\u4e00\u4e2a27\u00d727\u7684\u7f51\u683c\nN = torch.zeros((27, 27), dtype = torch.int32)\n\n# \u5c06\u59d3\u540d\u4e2d\u51fa\u73b0\u7684\u6240\u6709\u5b57\u6bcd\u505a\u6210\u4e00\u4e2a\u5217\u8868     chars = ['a','b','c','d','e' ...]\nchars = sorted(list(set(''.join(words))))\n# \u6211\u4eec\u5e0c\u671b\u628a\u7b2c0\u4e2a\u4f4d\u7f6e\u8ba9\u7ed9\u5f00\u542f\u6216\u7ed3\u675f\u5360\u4f4d\u7b26'.'       stoi = {'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5 ...}\nstoi = {s:i+1 for i, s in enumerate(chars)}\nstoi['.'] = 0\n# \u5efa\u7acb\u53cd\u5411\u7d22\u5f15        itos = {0: '.', 1: 'a', 2: 'b', 3: 'c', ...}\nitos = {i: s for s, i in stoi.items()}\n\nxs, ys = [], []\nfor word in words:\n    chs = ['.'] + list(word) + ['.']\n    for ch1, ch2 in zip(chs, chs[1:]):\n        ix1 = stoi[ch1]\n        ix2 = stoi[ch2]\n        # print(ch1, ch2, ix1, ix2)\n        xs.append(ix1)\n        ys.append(ix2)\n\nxs = torch.tensor(xs)\nys = torch.tensor(ys)\nnum = xs.nelement()\nprint(f\"\u6837\u672c\u6570\u91cf\uff1a{num}\")\n\nimport torch.nn.functional as F\ng = torch.Generator().manual_seed(2147483647)\nW = torch.randn((27, 27), generator=g, requires_grad=True)\nxenc = F.one_hot(xs, num_classes=27).float()\n\nfor i in range(100):\n    logits = xenc @ W\n    counts = logits.exp()\n    probs = counts \/ counts.sum(1, keepdim=True)\n    loss = -probs[torch.arange(num), ys].log().mean()\n    if i % 10 == 0:\n        print(loss.item())\n\n    W.grad = None\n    loss.backward()\n    W.data += -50 * W.grad\n\nlogits = xenc @ W\ncounts = logits.exp()\nprobs = counts \/ counts.sum(1, keepdim=True)\nloss = -probs[torch.arange(num), ys].log().mean()\nprint(loss.item())\n\nfor i in range(5):\n    out = []\n    ix = 0\n    while True:\n        xenc = F.one_hot(torch.tensor([ix]), num_classes=27).float()\n        logits = xenc @ W\n        counts = logits.exp()\n        probs = counts \/ counts.sum(1, keepdim=True) # \u4e0b\u4e00\u4e2a\u5b57\u7b26\u51fa\u73b0\u7684\u6982\u7387\uff0c\u8fd9\u4e2a\u5730\u65b9\u5e94\u8be5\u662f\u4e00\u4e2a\u670927\u4e2a\u5143\u7d20\u7684\u5217\u8868\n\n        # \u624b\u52a8\u4ece\u4e00\u4e2a\u5217\u8868\u4e2d\u968f\u673a\u51fa\u4e00\u4e2a\u4e0b\u6807\n        ix = torch.multinomial(probs, num_samples=1, replacement=True,generator=g).item()\n        out.append(itos[ix])\n        if ix == 0:\n            break\n\n    print(\"\".join(out))<\/code><\/pre>\n\n\n\n
\u8fd0\u884c\u5982\u4e0b\uff1a<\/p>\n\n\n\n
\u6837\u672c\u6570\u91cf\uff1a228146\n3.758953809738159\n2.6890034675598145\n2.572789430618286\n2.5301806926727295\n2.5086867809295654\n2.496137857437134\n2.4880495071411133\n2.482424736022949\n2.478290557861328\n2.475132942199707\n2.4726526737213135\njigla.\nsadryrolyiniydavesole.\nrish.\nbe.\nka.<\/code><\/pre>\n\n\n\n
\u53ef\u4ee5\u770b\u5230\u6211\u4eec\u8fd9\u4e2a\u6a21\u578b\u7684\u635f\u5931\u503c\u5df2\u7ecf\u903c\u8fd1\u4e86\u6837\u672c\u7684\u635f\u5931\u503c\u3002\u5e76\u4e14\u8f93\u51fa\u4e86\u540d\u5b57\uff0c\u867d\u7136\u5e76\u4e0d\u662f\u5f88\u6210\u529f\uff0c\u56e0\u4e3a\u6211\u4eec\u7684\u6a21\u578b\u53ea\u8003\u8651\u4e86\u4e0a\u4e00\u4e2a\u5b57\u7b26\u7684\u53ef\u80fd\u6027\u3002<\/p>\n\n\n\n
\u7ed3\u8bed<\/h2><\/div>\n\n\n\n
\u6211\u4eec\u8fd9\u6b21\u5b9e\u73b0\u7684\u662f\u4e00\u4e2a\u5355\u5c42\u7ebf\u6027\u6a21\u578b\uff0c\u5176\u8f93\u5165\u662f27\u4e2a\u8f93\u5165\u503c\uff0c\u800c\u5176\u8f93\u51fa\u4e5f\u5fc5\u987b\u662f27\u4e2a\u8f93\u51fa\u503c\uff0c\u56e0\u6b64\u5176\u53ef\u8bad\u7ec3\u53c2\u6570\u662f729\u4e2a\uff0c\u901a\u8fc7\u8fd9729\u4e2a\u53ef\u8bad\u7ec3\u53c2\u6570\uff0c\u6211\u4eec\u5c3d\u53ef\u80fd\u5730\u62df\u5408\u4e86\u90a3\u5f2027\u00d727\u7684\u6982\u7387\u5206\u5e03\u8868\u683c\uff0c\u5982\u679c\u6211\u4eec\u60f3\u589e\u52a0\u5176\u53c2\u6570\u7684\u8bdd\uff0c\u5c31\u53ea\u80fd\u589e\u52a0\u5176\u4e2d\u95f4\u5c42\uff08\u9690\u85cf\u5c42\uff09\uff0c\u4e0d\u8fc7\uff0c\u5bf9\u4e8e\u6211\u4eec\u7684\u8fd9\u4e2a\u76ee\u6807\uff0c\u53ef\u80fd\u5e76\u4e0d\u9700\u8981\u5982\u6b64\u590d\u6742\uff0c\u5e76\u4e14\u5f53\u524d\u5df2\u7ecf\u57fa\u672c\u4e0a\u8fbe\u5230\u4e86\u8fd9\u4e2a\u8bbe\u8ba1\u6a21\u578b\u7684\u903b\u8f91\u7684\u4e0a\u9650\uff0c\u5176\u635f\u5931\u503c\u5df2\u7ecf\u903c\u8fd1\u4e86\u6781\u9650\uff0c\u56e0\u6b64\u6211\u4eec\u5e76\u4e0d\u9700\u8981\u66f4\u52a0\u590d\u6742\u7684\u65b9\u5f0f\u6765\u5b9e\u73b0\u4e86\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"
\u8fd9\u662fAndrej Karpathy \u96f6\u57fa\u7840\u5b66\u4e60\u795e\u7ecf\u7f51\u7edc\u7684\u89c6\u9891\u8bfe\u7684\u7b2c\u4e8c\u8282\u8bfe\uff0c\u5728\u8fd9\u4e00\u8282\u8bfe\u4e2d\uff0c\u6211\u4eec\u4f1a\u5b9e\u73b0\u4e00\u4e2a\u7b80\u5355\u7684\u82f1 […]<\/p>\n","protected":false},"author":1,"featured_media":2406,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_uag_custom_page_level_css":"","footnotes":""},"categories":[10],"tags":[29,24,28],"class_list":["post-2406","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-10","tag-andrej-karpathy","tag-24","tag-28"],"uagb_featured_image_src":{"full":["https:\/\/pic.zjml.life\/api\/image\/1786270581915265baa?quality=mid",800,600,false],"thumbnail":["https:\/\/pic.zjml.life\/api\/image\/1786270581915265baa?quality=mid",150,150,true],"medium":["https:\/\/pic.zjml.life\/api\/image\/1786270581915265baa?quality=mid",300,300,false],"medium_large":["https:\/\/pic.zjml.life\/api\/image\/1786270581915265baa?quality=mid",768,0,false],"large":["https:\/\/pic.zjml.life\/api\/image\/1786270581915265baa?quality=mid",1024,1024,false],"1536x1536":["https:\/\/pic.zjml.life\/api\/image\/1786270581915265baa?quality=mid",1536,1536,false],"2048x2048":["https:\/\/pic.zjml.life\/api\/image\/1786270581915265baa?quality=mid",2048,2048,false]},"uagb_author_info":{"display_name":"zjml","author_link":"https:\/\/blog.zjml.life\/?author=1"},"uagb_comment_info":1,"uagb_excerpt":"\u8fd9\u662fAndrej Karpathy \u96f6\u57fa\u7840\u5b66\u4e60\u795e\u7ecf\u7f51\u7edc\u7684\u89c6\u9891\u8bfe\u7684\u7b2c\u4e8c\u8282\u8bfe\uff0c\u5728\u8fd9\u4e00\u8282\u8bfe\u4e2d\uff0c\u6211\u4eec\u4f1a\u5b9e\u73b0\u4e00\u4e2a\u7b80\u5355\u7684\u82f1…","_links":{"self":[{"href":"https:\/\/blog.zjml.life\/index.php?rest_route=\/wp\/v2\/posts\/2406","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/blog.zjml.life\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/blog.zjml.life\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/blog.zjml.life\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/blog.zjml.life\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=2406"}],"version-history":[{"count":11,"href":"https:\/\/blog.zjml.life\/index.php?rest_route=\/wp\/v2\/posts\/2406\/revisions"}],"predecessor-version":[{"id":2423,"href":"https:\/\/blog.zjml.life\/index.php?rest_route=\/wp\/v2\/posts\/2406\/revisions\/2423"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/blog.zjml.life\/index.php?rest_route=\/wp\/v2\/posts\/2406"}],"wp:attachment":[{"href":"https:\/\/blog.zjml.life\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=2406"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/blog.zjml.life\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=2406"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/blog.zjml.life\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=2406"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}