{ "dataset": "BabyLM 2026 English", "snapshot_utc": "2026-09-11T22:26:53.552288+00:00", "registered_updates": 12000, "registered_runs": 12, "runs": [ { "scale": "10m", "seed": 42, "variant": "flm", "complete": true, "parameters": 600003, "validation": [ { "step": 0, "presented_tokens": 0, "components": { "bnc_spoken": { "bits_per_byte": 3.6282229809216866, "token_perplexity": 4949.802183341031, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 5.146004276643903, "token_perplexity": 5181.768185882155, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 3.689196108253878, "token_perplexity": 5618.583019089439, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 4.333268936301915, "token_perplexity": 5120.981294907628, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 3.980398153351656, "token_perplexity": 5281.868758404689, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 4.5422665140606595, "token_perplexity": 4727.254366275884, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 4.156506363683319, "token_perplexity": 5139.336632547325, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "ccf13d8255072cf8ff8780c5630323778c3e3cce75f841649af35e60a07aa139" }, { "step": 500, "presented_tokens": 768000, "components": { "bnc_spoken": { "bits_per_byte": 2.240620955082518, "token_perplexity": 191.25147461690645, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.171860371274113, "token_perplexity": 36.95623191449404, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.3946363576028618, "token_perplexity": 271.5573611621873, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.4515980492286586, "token_perplexity": 125.49035589713814, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.7423350332294527, "token_perplexity": 367.15688407068905, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.3450744229070386, "token_perplexity": 78.90808636423648, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.397665229408353, "token_perplexity": 138.2365645787404, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "64f33add38a0c882cd002bf57e8ffaa1dde564d7cf146e1e761b08bbb3964864" }, { "step": 1000, "presented_tokens": 1536000, "components": { "bnc_spoken": { "bits_per_byte": 2.1190067476290926, "token_perplexity": 143.8023902255149, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.0594064017038516, "token_perplexity": 30.656064142605082, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.2796192212629665, "token_perplexity": 207.47243828938693, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.3391150032287547, "token_perplexity": 100.53630446013634, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.5745876615611616, "token_perplexity": 255.83554220667372, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.2636599344347235, "token_perplexity": 67.80452955974246, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.2780124616153654, "token_perplexity": 108.09299971475552, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "2fc305cfa3dede397687f7afcfa045e3cfc7aa76f9c4226389cd92978f0c5610" }, { "step": 1500, "presented_tokens": 2304000, "components": { "bnc_spoken": { "bits_per_byte": 2.0581521837162686, "token_perplexity": 124.68047055320888, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.063965115492753, "token_perplexity": 30.88922109832238, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.2131602606319043, "token_perplexity": 177.58767548211338, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.28743649158477, "token_perplexity": 90.79986798519586, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.481899589989964, "token_perplexity": 209.54222949890456, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.196445008580215, "token_perplexity": 59.82505196628549, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.219856627509441, "token_perplexity": 95.91275056445012, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "717cbdf363320296bb60d587b8754325d5b107240763b70ac2f839a2c0a01c63" }, { "step": 2000, "presented_tokens": 3072000, "components": { "bnc_spoken": { "bits_per_byte": 2.0109037512387067, "token_perplexity": 111.60554321168085, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9792452354290877, "token_perplexity": 26.83209801487483, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.1758656696189926, "token_perplexity": 162.7448781165536, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.2465093235909674, "token_perplexity": 83.76273572120087, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.4153511619018047, "token_perplexity": 181.56416340168857, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.1067252456449883, "token_perplexity": 50.61759051060408, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.160673951738994, "token_perplexity": 84.92554908415319, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "e107e649e75ff05865dc8afcf71d8e29db29a69acb9dc02b03ba41735c1f1604" }, { "step": 2500, "presented_tokens": 3840000, "components": { "bnc_spoken": { "bits_per_byte": 1.985549390649379, "token_perplexity": 105.16413189702548, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.98235506419329, "token_perplexity": 26.97114362767027, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.154904155167607, "token_perplexity": 154.95388020794275, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.218621564880365, "token_perplexity": 79.28270316371481, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.3798750864400144, "token_perplexity": 168.2093225391683, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.061831889033611, "token_perplexity": 46.55684898332779, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.134865736985415, "token_perplexity": 80.53727384044565, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "366dcf69a54010f2971efa3cd99b3f4bd8b6a715c1deefc92de350af0617a083" }, { "step": 3000, "presented_tokens": 4608000, "components": { "bnc_spoken": { "bits_per_byte": 1.9650900331250363, "token_perplexity": 100.23838381233729, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9291369404264371, "token_perplexity": 24.687980609919016, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.1337012364932684, "token_perplexity": 147.45252839204932, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1975348600464524, "token_perplexity": 76.05501650188174, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.3471650615496613, "token_perplexity": 156.76785745383086, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.0224123938902747, "token_perplexity": 43.26073980614187, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.1047485363864187, "token_perplexity": 75.70219047528582, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "31432898c403301c1c7e28f3d2bbf214af62f28c5e0f42bccc434ffac8289995" }, { "step": 3500, "presented_tokens": 5376000, "components": { "bnc_spoken": { "bits_per_byte": 1.9666810611307075, "token_perplexity": 100.61302016147167, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9240646106376613, "token_perplexity": 24.480724653402902, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.1130629622247854, "token_perplexity": 140.49985801980662, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.193808928436891, "token_perplexity": 75.49851387184445, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.3211249678847543, "token_perplexity": 148.21844024020518, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.9515240417315092, "token_perplexity": 37.909371654267396, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0846040950312696, "token_perplexity": 72.6312627332094, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "6cfd8a847389e716a4fec89b953104a22288bfc53edd4a6d43378f17cd76ee81" }, { "step": 4000, "presented_tokens": 6144000, "components": { "bnc_spoken": { "bits_per_byte": 1.941751594842359, "token_perplexity": 94.90054956242346, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.922664646118713, "token_perplexity": 24.423828950068376, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0968008607810797, "token_perplexity": 135.2531546443308, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.175905132591213, "token_perplexity": 72.88068467239925, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.3013450318998347, "token_perplexity": 142.0373156770341, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.9555381591440495, "token_perplexity": 38.193893455549244, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0708826032304133, "token_perplexity": 70.61112303808144, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "1c93d4c085ce0c24cd846d399653b9b8bfd5d227e4be5812d10643d719bb79da" }, { "step": 4500, "presented_tokens": 6912000, "components": { "bnc_spoken": { "bits_per_byte": 1.93615045716626, "token_perplexity": 93.66236889971596, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9161942480241227, "token_perplexity": 24.16257975940222, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.088198548520666, "token_perplexity": 132.55746232391036, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.157297572427733, "token_perplexity": 70.25610202858145, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2873691397120277, "token_perplexity": 137.82597733239118, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.990945880744875, "token_perplexity": 40.79792913596348, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0672474185852043, "token_perplexity": 70.08541431313164, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "4e53cd190cfe9fae12e3f5befb56ba3b32ec3f10b503b4639d9a3d9f72426c25" }, { "step": 5000, "presented_tokens": 7680000, "components": { "bnc_spoken": { "bits_per_byte": 1.9252979904669998, "token_perplexity": 91.3091217254636, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.938485252125798, "token_perplexity": 25.074561466494128, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.077273119160917, "token_perplexity": 129.21108953394912, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.148130445987751, "token_perplexity": 68.9980523165467, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2696905665165383, "token_perplexity": 132.6773110582289, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.9209280103168895, "token_perplexity": 35.809228699367885, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.050989461907733, "token_perplexity": 67.78172648241775, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "45cd02faa7d9fb470919d25b812287a86aa090dc8e9530e389a7618c5ee97816" }, { "step": 5500, "presented_tokens": 8448000, "components": { "bnc_spoken": { "bits_per_byte": 1.9172731985263207, "token_perplexity": 89.6071345591069, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9153681401342435, "token_perplexity": 24.12942657196093, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0693241033480905, "token_perplexity": 126.82958292266218, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.14148193687593, "token_perplexity": 68.09976178870406, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2581613884438285, "token_perplexity": 129.42365164738385, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.9212371911724528, "token_perplexity": 35.82985811069948, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.041812049373235, "token_perplexity": 66.51492119914859, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "c7686130f66fd2b606c1a5ebc7e58fb404351d263b9a3a626b21784e3d2c881c" }, { "step": 6000, "presented_tokens": 9216000, "components": { "bnc_spoken": { "bits_per_byte": 1.9239238884717564, "token_perplexity": 91.0154099076176, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9231606593882697, "token_perplexity": 24.443972193759603, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0652148592374795, "token_perplexity": 125.61572489927985, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1390858369317582, "token_perplexity": 67.77889585827866, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.241893387165256, "token_perplexity": 124.96791084431197, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.9252053941310028, "token_perplexity": 36.09568521590589, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.040722684347134, "token_perplexity": 66.36613123905434, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "a47c7d2aef66bd3cfebadfc93bdc0ed6767c1de11d3725f9c9c2dfb7472eff89" }, { "step": 6500, "presented_tokens": 9984000, "components": { "bnc_spoken": { "bits_per_byte": 1.9115600313601386, "token_perplexity": 88.4147903409024, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.893631523976906, "token_perplexity": 23.27325471124007, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0541507189523256, "token_perplexity": 122.4048519684839, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1252624328150467, "token_perplexity": 65.95707938979261, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2288949441430974, "token_perplexity": 121.51819865828703, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8956874569348097, "token_perplexity": 34.16455980522829, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0233577818225212, "token_perplexity": 64.03880331192174, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "ba5d27c904b4dcace10d7b3cedd8579f0a0ccac8e96c95bc8a994c989d02ae80" }, { "step": 7000, "presented_tokens": 10752000, "components": { "bnc_spoken": { "bits_per_byte": 1.9100807341479675, "token_perplexity": 88.10865383860873, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9082194793774871, "token_perplexity": 23.844431361142394, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0449877911284173, "token_perplexity": 119.80794314960829, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.125288106580074, "token_perplexity": 65.96041718894152, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.221232640533858, "token_perplexity": 119.52945028127195, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8599901183935785, "token_perplexity": 31.9666663165351, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0165789870573994, "token_perplexity": 63.15258547239749, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "2e8b2ec80726961c65af800745281fc4c0725fa0e39beb9bb49a67c93df86bdd" }, { "step": 7500, "presented_tokens": 11520000, "components": { "bnc_spoken": { "bits_per_byte": 1.8993645747746501, "token_perplexity": 85.92239899859001, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8783294941584254, "token_perplexity": 22.68881655002639, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.045254111775503, "token_perplexity": 119.88263922600497, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.119787466513013, "token_perplexity": 65.24913381130706, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.215136439323351, "token_perplexity": 117.97045809761136, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8700246189688992, "token_perplexity": 32.56979849731031, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.010136821372744, "token_perplexity": 62.321744796304124, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "65fba166f311be1ab2f405e9083c666f5686e46f88fbc778a02752144ef4fa5a" }, { "step": 8000, "presented_tokens": 12288000, "components": { "bnc_spoken": { "bits_per_byte": 1.8943521632978646, "token_perplexity": 84.91849709511776, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8801399787346682, "token_perplexity": 22.757192534290173, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.039209293046375, "token_perplexity": 118.19863462921465, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1118915221919363, "token_perplexity": 64.24150190523495, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.207521740555116, "token_perplexity": 116.05166850953478, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.870522699349765, "token_perplexity": 32.6000307284126, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0057649515490104, "token_perplexity": 61.76414319392806, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "1df7bfd71f6f91b1526f6f761af722e46353b4f502c6b120e75a120b19ed7151" }, { "step": 8500, "presented_tokens": 13056000, "components": { "bnc_spoken": { "bits_per_byte": 1.8952032348652, "token_perplexity": 85.08812194969464, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8774237430755272, "token_perplexity": 22.654686479399153, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0345813167781213, "token_perplexity": 116.92534909446512, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1055489427150187, "token_perplexity": 63.44338228879798, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.202439833826878, "token_perplexity": 114.78850033619311, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8578710069007653, "token_perplexity": 31.840730726040753, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.000871120256399, "token_perplexity": 61.14588487317782, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "6ac7df8dd89750709e8e00fb2f8a70e936a9e585855f7db4ba8fcb0b3731cf75" }, { "step": 9000, "presented_tokens": 13824000, "components": { "bnc_spoken": { "bits_per_byte": 1.8885327904305682, "token_perplexity": 83.76767905675065, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8789657599583058, "token_perplexity": 22.71282274958251, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.029840868793266, "token_perplexity": 115.63533863857803, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1033702523484172, "token_perplexity": 63.17152066366746, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.195967789628917, "token_perplexity": 113.19968209829537, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8663857157502974, "token_perplexity": 32.34977517762605, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9987191830613602, "token_perplexity": 60.87598425542775, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "beea3b9cacee5ef5671139cf9a01189572199bc54cca776174933ce506d916ec" }, { "step": 9500, "presented_tokens": 14592000, "components": { "bnc_spoken": { "bits_per_byte": 1.8807317095203122, "token_perplexity": 82.2493945911705, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8726284419566523, "token_perplexity": 22.474845873761087, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0281668410177094, "token_perplexity": 115.18319671145146, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0952035380693217, "token_perplexity": 62.16278775446544, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1936964886279937, "token_perplexity": 112.64733050258609, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8432253850174622, "token_perplexity": 30.98382212741526, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9908653736417405, "token_perplexity": 59.90101152519588, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "668704b3ef14e5278a349e05d548fd3a9c16ccb21ac7e697aeaa4ee9fbf93eb6" }, { "step": 10000, "presented_tokens": 15360000, "components": { "bnc_spoken": { "bits_per_byte": 1.8821820387247412, "token_perplexity": 82.52956718830364, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8820768988130245, "token_perplexity": 22.83057173187086, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.024971746173439, "token_perplexity": 114.32512779601844, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0950195711033, "token_perplexity": 62.140251080400866, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1914784645710417, "token_perplexity": 112.11053636467834, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8478109047450824, "token_perplexity": 31.249609218750564, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9920838578835411, "token_perplexity": 60.05124455711899, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "7d7a12669dd0e1bd7132158189c15ac9f58d54af1c6f380701048ff26acd29e3" }, { "step": 10500, "presented_tokens": 16128000, "components": { "bnc_spoken": { "bits_per_byte": 1.879149420772921, "token_perplexity": 81.94481500034, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8733652014491735, "token_perplexity": 22.502383831334924, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.021706231749074, "token_perplexity": 113.45475191445195, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.09072681042775, "token_perplexity": 61.61668455764854, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1862410083328903, "token_perplexity": 110.85312515681812, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8486266885089275, "token_perplexity": 31.29713222154568, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9882366568724361, "token_perplexity": 59.57818267066635, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "58e5618a28d4c902cc9a4e3f4c12dd11769f58239abe4a652d5d718b23e6240f" }, { "step": 11000, "presented_tokens": 16896000, "components": { "bnc_spoken": { "bits_per_byte": 1.877839830090025, "token_perplexity": 81.69358114107419, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8646971629358338, "token_perplexity": 22.18052303771551, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0187095259530716, "token_perplexity": 112.66185489832526, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0897548219824285, "token_perplexity": 61.498749830216575, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1845515450463897, "token_perplexity": 110.45053392634048, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.845143642647855, "token_perplexity": 31.094732529761398, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9852748034779844, "token_perplexity": 59.216525871612745, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "8b4771a3ac1627a886157d9567f992b5d44dcfcd973fc34c41da1dc04c756c5f" }, { "step": 11500, "presented_tokens": 17664000, "components": { "bnc_spoken": { "bits_per_byte": 1.87467042022472, "token_perplexity": 81.08873994494607, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.867797079167429, "token_perplexity": 22.295096580693926, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.017162054128495, "token_perplexity": 112.25458204175531, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.089670651760947, "token_perplexity": 61.488547791568266, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1833214252299054, "token_perplexity": 110.15832226728314, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.843124894589773, "token_perplexity": 30.97802285852553, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9842633139989003, "token_perplexity": 59.09352160852924, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "fb33cfe1f1bcd01344e272ae2d702964814dc0f27e8ea4ce7f7c15ef74f75aae" }, { "step": 12000, "presented_tokens": 18432000, "components": { "bnc_spoken": { "bits_per_byte": 1.875806796282298, "token_perplexity": 81.30508603560565, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.871258128742138, "token_perplexity": 22.42371709286763, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.01496680462222, "token_perplexity": 111.67934766737407, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0884060988183397, "token_perplexity": 61.33547839936618, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1790216355944145, "token_perplexity": 109.14297933297752, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8267943431377658, "token_perplexity": 30.049869068787572, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9811101259407082, "token_perplexity": 58.71170901599371, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "551cab07209c4bde330c4502d50de9197e1293bfa588bc22e37553eb8f936222" } ], "source_commit": "518a55bbeb39d7f9653b68603deff57e370b562e", "protocol": { "steps": 12000, "batch": 16, "sequence": 96, "warmup": 16, "learning_rate": 0.002, "final_learning_rate": 0.0002, "lr_warmup_updates": 100, "weight_decay": 0.01, "eval_tokens": 49152, "threads": 4, "tokenizer_sha256": "46a2ef1bf8dce00e801c5ac188b13ec5c30c7f1ab7e8184031dca3ee057a7242", "train_cache_sha256": "4d205d6d35c87c38749ca3dc858fe96b28e3d7f4b8d7e2eb782e06909c8f15cd", "validation_cache_sha256": "ea3bd0f2625e78723baf6300dca2ffdae91deebdcb21ae748ffd0cc4ba2ef859", "graph_sha256": "a2f35369a5ed4981b8ef75c92df285f7df576a020582f119a1372bfbedbaebee", "validation_panel_sha256": "9a0c030150fee1821246dd54f045fc36c437bab3061e3010740d2942900de3ae", "evaluation_unit": "block", "cache_identity": "SHA-256 of verified mmap manifest" } }, { "scale": "10m", "seed": 42, "variant": "gru", "complete": true, "parameters": 595408, "validation": [ { "step": 0, "presented_tokens": 0, "components": { "bnc_spoken": { "bits_per_byte": 3.548910474352327, "token_perplexity": 4109.837103544142, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 5.004167969041716, "token_perplexity": 4093.5301659483216, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 3.5565584390605474, "token_perplexity": 4119.234046030016, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 4.222534801199459, "token_perplexity": 4116.8292430438205, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 3.864499819140937, "token_perplexity": 4115.192808120306, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 4.468084879518217, "token_perplexity": 4117.156841240587, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 4.048016914079572, "token_perplexity": 4111.954062705375, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "8dfafa760598d1acf5e61c093c4ba5d35f4b422099d1585258b0ce23dac3fa4f" }, { "step": 500, "presented_tokens": 768000, "components": { "bnc_spoken": { "bits_per_byte": 2.280625553724639, "token_perplexity": 210.05891918236574, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.235110344506052, "token_perplexity": 41.05278704606856, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.414914415797802, "token_perplexity": 284.7552905594689, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.482472634114101, "token_perplexity": 133.36428160223215, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.7618704218314636, "token_perplexity": 382.9329503081316, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.5094295432252505, "token_perplexity": 107.17210010418395, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.450805847540343, "token_perplexity": 154.19368843850685, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "aeafa52fac06c1b0983bc8f2513e06b0548d68c76913b2f0a90c9b2e7c3b8222" }, { "step": 1000, "presented_tokens": 1536000, "components": { "bnc_spoken": { "bits_per_byte": 2.1482151854667237, "token_perplexity": 153.99577383308088, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.1134105293493515, "token_perplexity": 33.53494063607625, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.2732258479771428, "token_perplexity": 204.39125548079235, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.3360279474992893, "token_perplexity": 99.92642360215065, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.5769597578557075, "token_perplexity": 257.145810964527, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.338190220644028, "token_perplexity": 77.90266590085717, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.300805187879233, "token_perplexity": 113.27831898457003, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "d616c45b5bc6d8eee341a722194cae230b338ef66098e937e9c2e3a3bbda4892" }, { "step": 1500, "presented_tokens": 2304000, "components": { "bnc_spoken": { "bits_per_byte": 2.0855878684535027, "token_perplexity": 132.96457548650847, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.085627912700368, "token_perplexity": 32.02163861996616, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.1881478510409047, "token_perplexity": 167.49070032824162, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.2604483399753166, "token_perplexity": 86.09598367093926, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.4798716064649016, "token_perplexity": 208.62907262284966, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.2136714703529567, "token_perplexity": 61.77588433191784, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.221179966051235, "token_perplexity": 96.17403015874132, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "17ddc715711e240468941d021dba8c04df4ecfead614ba3cec726d5441943d8c" }, { "step": 2000, "presented_tokens": 3072000, "components": { "bnc_spoken": { "bits_per_byte": 2.01012482363471, "token_perplexity": 111.40189807219741, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.0134740297481617, "token_perplexity": 28.402828272161955, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.139926009650598, "token_perplexity": 149.616317619418, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.209956559563701, "token_perplexity": 77.9401172198916, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.3914265394959227, "token_perplexity": 172.44631052336823, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.0710781483178238, "token_perplexity": 47.365664078175655, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.142917267216697, "token_perplexity": 81.88140549621286, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "0a51cce09c5415e5cb68417ac1cfcb8f6f7a2c0578039fad86c6907112f9679e" }, { "step": 2500, "presented_tokens": 3840000, "components": { "bnc_spoken": { "bits_per_byte": 1.9751295549692494, "token_perplexity": 102.62595542319272, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.008728940173767, "token_perplexity": 28.17970857211062, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.103468442076257, "token_perplexity": 137.38021846347212, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.170694966681029, "token_perplexity": 72.13606562473966, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.3457664392781536, "token_perplexity": 156.29637991610352, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.9951051881064645, "token_perplexity": 41.115248685091174, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.1029884320801187, "token_perplexity": 75.42877146951041, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "caf5e25a27c205c9c7848d908f5f4a956b758e456d90590b030d23789a9571d4" }, { "step": 3000, "presented_tokens": 4608000, "components": { "bnc_spoken": { "bits_per_byte": 1.9539702599934912, "token_perplexity": 97.65868849631154, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9559322643449828, "token_perplexity": 25.812312360352227, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.077693045193295, "token_perplexity": 129.33813465713027, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.153845233491743, "token_perplexity": 69.77965029008521, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.298290968580666, "token_perplexity": 141.1061852783673, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.915481061356518, "token_perplexity": 35.44773469169168, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.063954733613298, "token_perplexity": 69.6126157730561, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "f07f6ab77b71ceaa946d67da1b9fb828c3d474f4afc4574f6e2a1b8e7f629db1" }, { "step": 3500, "presented_tokens": 5376000, "components": { "bnc_spoken": { "bits_per_byte": 1.9418551456392668, "token_perplexity": 94.92359381423012, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9613566509814977, "token_perplexity": 26.04607767043111, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0500676869899417, "token_perplexity": 121.24077928052412, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1354099402469937, "token_perplexity": 67.28958635249491, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2671533011437357, "token_perplexity": 131.95431821403002, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.853833884763864, "token_perplexity": 31.6021819641866, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.039448548395367, "token_perplexity": 66.19252675355831, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "00db1566347dc730eff4851ddf2e4b79820bed5952200a14a53c467857a663a2" }, { "step": 4000, "presented_tokens": 6144000, "components": { "bnc_spoken": { "bits_per_byte": 1.9196902981485895, "token_perplexity": 90.11641393957281, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9342881759920136, "token_perplexity": 24.90025654649758, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.023072419993583, "token_perplexity": 113.81808060598394, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1163401209580006, "token_perplexity": 64.8072753833358, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.233245625291028, "token_perplexity": 122.66210943781074, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8538896696705263, "token_perplexity": 31.605466020170784, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0175065162615655, "token_perplexity": 63.27311686032307, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "2e1239f38b7322c6e93330c80d0a313fc28f00a3fa6243386c379193734b6580" }, { "step": 4500, "presented_tokens": 6912000, "components": { "bnc_spoken": { "bits_per_byte": 1.8999980347593466, "token_perplexity": 86.05011221990941, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9126258080099485, "token_perplexity": 24.01969767082091, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0159469769208074, "token_perplexity": 111.93582293629872, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.095392678911357, "token_perplexity": 62.18596677015133, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.214600222338282, "token_perplexity": 117.834307192799, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.824021404900403, "token_perplexity": 29.89505306859853, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9983785989757135, "token_perplexity": 60.833376796191445, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "f4efea5ddafddc391fea09e6123d39cf430572fcd6a57a903b3ecf91bc450957" }, { "step": 5000, "presented_tokens": 7680000, "components": { "bnc_spoken": { "bits_per_byte": 1.8905855032589398, "token_perplexity": 84.1718247394747, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9473173567822784, "token_perplexity": 25.44535518806204, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0043067599319553, "token_perplexity": 108.92766836236558, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0878179193631703, "token_perplexity": 61.26441137716104, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1915865112997475, "token_perplexity": 112.13662588009946, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8045002473711986, "token_perplexity": 28.827506318884968, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.99087869426228, "token_perplexity": 59.90265185764614, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "58e0db8537b27de1f4bff507162f4a0fd7d2cb1cccb12b1338a619e474df26ab" }, { "step": 5500, "presented_tokens": 8448000, "components": { "bnc_spoken": { "bits_per_byte": 1.879509142815921, "token_perplexity": 82.0139597820532, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.917252841506345, "token_perplexity": 24.20512960243116, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9922307323520838, "token_perplexity": 105.89229702089347, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0800220789116945, "token_perplexity": 60.330217481366766, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1821758524413624, "token_perplexity": 109.88688977881444, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7962933534529721, "token_perplexity": 28.39016076086745, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9783024355977366, "token_perplexity": 58.3738087723826, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "44bf1cee8d575803b479ad0c22780ac529ba32f0604d73073778a3110f65b9ea" }, { "step": 6000, "presented_tokens": 9216000, "components": { "bnc_spoken": { "bits_per_byte": 1.8775032880140599, "token_perplexity": 81.62914289933323, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9418405822148135, "token_perplexity": 25.21478582542386, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9859159859601494, "token_perplexity": 104.33888331010641, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0704297486934546, "token_perplexity": 59.20027037976845, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1560747270177227, "token_perplexity": 103.88049618905812, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7903296471473944, "token_perplexity": 28.076523049538018, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.97304545056487, "token_perplexity": 57.74636112130896, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "f3d2a75c3c430b58afa3e91a50559039e39b8c25cf6d53706186123273eb408e" }, { "step": 6500, "presented_tokens": 9984000, "components": { "bnc_spoken": { "bits_per_byte": 1.8648574764214867, "token_perplexity": 79.24431519479768, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9089934974408456, "token_perplexity": 23.875125876452955, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9727004358951015, "token_perplexity": 101.16124314349477, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.06201212552523, "token_perplexity": 58.226147894653764, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1459186130215406, "token_perplexity": 101.63310711807112, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.769727246574369, "token_perplexity": 27.019441408095382, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9574781992237227, "token_perplexity": 55.927610188751366, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "ca94b38a31ca8f7b516d88c1109d9f1221fd9c1640c15a4c7e8d78a402f62d0a" }, { "step": 7000, "presented_tokens": 10752000, "components": { "bnc_spoken": { "bits_per_byte": 1.8575059298971168, "token_perplexity": 77.8900712226752, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9139656383655694, "token_perplexity": 24.073245803451208, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9680967775346911, "token_perplexity": 100.07718693096005, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0564678113296173, "token_perplexity": 57.59330884208871, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.132824099728405, "token_perplexity": 98.80710250529461, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7533115968411606, "token_perplexity": 26.205739015991455, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9502258554847873, "token_perplexity": 55.09997574893552, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "0db577ef494a9ee5f36ffc7fd061b3ca1d2982d49fd47c927965886386c52e2f" }, { "step": 7500, "presented_tokens": 11520000, "components": { "bnc_spoken": { "bits_per_byte": 1.8484932244280516, "token_perplexity": 76.26135922104349, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.894669643512469, "token_perplexity": 23.313445130892738, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9649872959885466, "token_perplexity": 99.35155595306105, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0502173967757464, "token_perplexity": 56.888118103405674, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1279967500274237, "token_perplexity": 97.78522497377945, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7369846531890967, "token_perplexity": 25.42074173320358, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9410151250141339, "token_perplexity": 54.06648250817056, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "8ba9612cead87c30b4832e4d51a1fec131979003bf832cfadf46778bdc5773c0" }, { "step": 8000, "presented_tokens": 12288000, "components": { "bnc_spoken": { "bits_per_byte": 1.844375885067425, "token_perplexity": 75.5286793127395, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8956011895940126, "token_perplexity": 23.3495686772718, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9596778739055314, "token_perplexity": 98.12468604617072, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.045032136345439, "token_perplexity": 56.30965864436661, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1189289420296733, "token_perplexity": 95.89419062681615, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7372567730997326, "token_perplexity": 25.433630543151907, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.936979835644991, "token_perplexity": 53.619829751273464, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "21138e7d03552dac42ac3a7c516f9b732eef61c7f14aef07e2b11ec5fabf50fd" }, { "step": 8500, "presented_tokens": 13056000, "components": { "bnc_spoken": { "bits_per_byte": 1.84483212423832, "token_perplexity": 75.60951897468775, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8867423451944, "token_perplexity": 23.008292513001408, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9562212640542735, "token_perplexity": 97.33410717351471, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.040180356085429, "token_perplexity": 55.773729591435455, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.111659421604398, "token_perplexity": 94.40462237672669, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.725287606803867, "token_perplexity": 24.872848568747592, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.931364846382241, "token_perplexity": 53.00445797660295, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "31945618a3f96ee6668cfe85f0b3267a3651c315803b8e5b3ea4744ca4646f3b" }, { "step": 9000, "presented_tokens": 13824000, "components": { "bnc_spoken": { "bits_per_byte": 1.8360973789778139, "token_perplexity": 74.07675682527852, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8819731349372126, "token_perplexity": 22.826634700657756, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9507085804273492, "token_perplexity": 96.08643172198668, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.031417047124517, "token_perplexity": 54.81862557952205, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.105998762308319, "token_perplexity": 93.2607613961166, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7372734761735278, "token_perplexity": 25.43442188789489, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9274371982401861, "token_perplexity": 52.57821211310679, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "7a2e61e60c8dee047a33a087bd86cdd8570f8bade1d8147e91430eeed98a3912" }, { "step": 9500, "presented_tokens": 14592000, "components": { "bnc_spoken": { "bits_per_byte": 1.8299964122897987, "token_perplexity": 73.02463567305435, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8717151979180715, "token_perplexity": 22.440758201481888, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.946069313531332, "token_perplexity": 95.04883660672202, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.031225016471916, "token_perplexity": 54.79788051717039, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1027457567671517, "token_perplexity": 92.60970165066696, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7186796772212163, "token_perplexity": 24.56856742854048, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9206082538885703, "token_perplexity": 51.84524945672207, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "6c78d811aef263a61db00df1b6ebab4d545e584b0196370d38d998efd152b196" }, { "step": 10000, "presented_tokens": 15360000, "components": { "bnc_spoken": { "bits_per_byte": 1.8292724075200624, "token_perplexity": 72.90077598831083, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8726634871581398, "token_perplexity": 22.476154999210504, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9438439503783602, "token_perplexity": 94.55510688359423, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0288224772964836, "token_perplexity": 54.53899689913779, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.095507696706824, "token_perplexity": 91.1773300915581, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.714388694161197, "token_perplexity": 24.372973065440323, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.917880466429177, "token_perplexity": 51.55533511102458, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "899111bd0a29216caf7a7304c9b604b680c193fb272cda1bd21a2a2e377372ad" }, { "step": 10500, "presented_tokens": 16128000, "components": { "bnc_spoken": { "bits_per_byte": 1.8258587814216507, "token_perplexity": 72.31961166809805, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8655857494795112, "token_perplexity": 22.213305048406063, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9392845392447835, "token_perplexity": 93.55152976904802, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.022595999597299, "token_perplexity": 53.87374587108862, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0899943745666363, "token_perplexity": 90.10115738849986, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7183730482241493, "token_perplexity": 24.554538540769833, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9140568542529008, "token_perplexity": 51.15168132078446, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "8b004adf2eca151da36081964f485df8e06c1d8c76f68bd82b0cfc394f505144" }, { "step": 11000, "presented_tokens": 16896000, "components": { "bnc_spoken": { "bits_per_byte": 1.8240924264656213, "token_perplexity": 72.02071383891574, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.859479317840802, "token_perplexity": 21.98899846908485, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9369875722135284, "token_perplexity": 93.04998365054391, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.021034675961582, "token_perplexity": 53.708206900153286, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0900031871706513, "token_perplexity": 90.10286738876934, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7161266067919285, "token_perplexity": 24.45200339890888, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.911875204080991, "token_perplexity": 50.922784993647404, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "6fe9003d916c517ad34ce0976c9ce7c8412eb1a5cbeec93206bd0c9ba77d01e9" }, { "step": 11500, "presented_tokens": 17664000, "components": { "bnc_spoken": { "bits_per_byte": 1.82132466658739, "token_perplexity": 71.55484344057516, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8667091179161885, "token_perplexity": 22.254818058863403, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9358248866704226, "token_perplexity": 92.79713546939828, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.021351454198609, "token_perplexity": 53.74175205599635, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.087216094982718, "token_perplexity": 89.56367385691007, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7062972458413783, "token_perplexity": 24.008370117465294, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9101822371579678, "token_perplexity": 50.74586683087597, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "16d674a04a9857666076d6b4ca26cf2ababe09989b8a9b8db7e058a11119d664" }, { "step": 12000, "presented_tokens": 18432000, "components": { "bnc_spoken": { "bits_per_byte": 1.82132391825712, "token_perplexity": 71.55471788971762, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8720809036900188, "token_perplexity": 22.454402294400392, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.934295131564724, "token_perplexity": 92.46550738770635, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0182948662201095, "token_perplexity": 53.4189477163564, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0836140362859514, "token_perplexity": 88.87159315557543, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.700490977805666, "token_perplexity": 23.75010396024177, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9086214616394719, "token_perplexity": 50.583307502204754, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "ecb1da92c942ae0e84f5deba5326333084f01e94ed6a9f50168fc9c879f85fa0" } ], "source_commit": "f7f7b192949bca2e98246e8bdb45633c3940a56c", "protocol": { "steps": 12000, "batch": 16, "sequence": 96, "warmup": 16, "learning_rate": 0.002, "final_learning_rate": 0.0002, "lr_warmup_updates": 100, "weight_decay": 0.01, "eval_tokens": 49152, "threads": 4, "tokenizer_sha256": "46a2ef1bf8dce00e801c5ac188b13ec5c30c7f1ab7e8184031dca3ee057a7242", "train_cache_sha256": "4d205d6d35c87c38749ca3dc858fe96b28e3d7f4b8d7e2eb782e06909c8f15cd", "validation_cache_sha256": "ea3bd0f2625e78723baf6300dca2ffdae91deebdcb21ae748ffd0cc4ba2ef859", "graph_sha256": "a2f35369a5ed4981b8ef75c92df285f7df576a020582f119a1372bfbedbaebee", "validation_panel_sha256": "9a0c030150fee1821246dd54f045fc36c437bab3061e3010740d2942900de3ae", "evaluation_unit": "block", "cache_identity": "SHA-256 of verified mmap manifest" } }, { "scale": "10m", "seed": 42, "variant": "transformer", "complete": true, "parameters": 607468, "validation": [ { "step": 0, "presented_tokens": 0, "components": { "bnc_spoken": { "bits_per_byte": 3.5717591360341667, "token_perplexity": 4336.0192509564895, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 5.039900792101366, "token_perplexity": 4344.007698997058, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 3.5868896909989223, "token_perplexity": 4422.262924176443, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 4.243492886783186, "token_perplexity": 4290.454865886343, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 3.889923762241634, "token_perplexity": 4346.790394255423, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 4.489396133784483, "token_perplexity": 4283.885151836662, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 4.073936771879149, "token_perplexity": 4336.999451555523, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "673db68728bce4e952d26cad84bc56113a79576c693c26fac53fdd51b13dcd16" }, { "step": 500, "presented_tokens": 768000, "components": { "bnc_spoken": { "bits_per_byte": 2.260845832611321, "token_perplexity": 200.53935276699772, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.2078960583472695, "token_perplexity": 39.23727945553403, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.4018493177649383, "token_perplexity": 276.18029648283374, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.4706388314969656, "token_perplexity": 130.2895517928645, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.747379189941512, "token_perplexity": 371.1670160263253, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.312919935910586, "token_perplexity": 74.32058687282523, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.4068016500281453, "token_perplexity": 140.85746905386537, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "5fdb93df5a5e62e3dd7552a5715c025c7a6cd0171422dd5455849a05adeae620" }, { "step": 1000, "presented_tokens": 1536000, "components": { "bnc_spoken": { "bits_per_byte": 2.12636908933397, "token_perplexity": 146.30632889587986, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.026860536886258, "token_perplexity": 29.041845348364742, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.2617828516563563, "token_perplexity": 198.99028479549767, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.3443010621842184, "token_perplexity": 101.56925652448555, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.5744124404577926, "token_perplexity": 255.73902099080718, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.3168660120960904, "token_perplexity": 74.86889657688141, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.2804756041088123, "token_perplexity": 108.64172431056294, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "7b52d492e2cfd3dcfe20ed7ae1d557ae4a5a3de72abc2ecf23f106b524e2d857" }, { "step": 1500, "presented_tokens": 2304000, "components": { "bnc_spoken": { "bits_per_byte": 2.073687626570696, "token_perplexity": 129.30580987985448, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.0504423522167894, "token_perplexity": 30.20271501264091, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.184374505095992, "token_perplexity": 166.0181398257717, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.282085534822018, "token_perplexity": 89.84723273363872, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.4786733811999597, "token_perplexity": 208.09140917283386, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.120361769483947, "token_perplexity": 51.91981804540667, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.2026174115459742, "token_perplexity": 92.57319741876374, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "de75f65500cff561c4d885f249b63b2c1a973969c01726adb4cb0ba644ea7cdb" }, { "step": 2000, "presented_tokens": 3072000, "components": { "bnc_spoken": { "bits_per_byte": 2.0043825804215953, "token_perplexity": 109.91205155005996, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9498264793463318, "token_perplexity": 25.55169101359234, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.139159211592688, "token_perplexity": 149.34806600979417, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.2173522283468206, "token_perplexity": 79.08459138944956, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.40351002162556, "token_perplexity": 176.99269769715931, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.925707142308485, "token_perplexity": 36.129437119330625, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.1144413776091873, "token_perplexity": 77.22575518850773, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "c4f634eecfd77cc2dc424667b00258917d6a0c4ceb567704f0b2860613b97ec0" }, { "step": 2500, "presented_tokens": 3840000, "components": { "bnc_spoken": { "bits_per_byte": 1.966312359918874, "token_perplexity": 100.52607833544606, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9266091426285383, "token_perplexity": 24.584476106185377, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.1035987292032523, "token_perplexity": 137.422113629335, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.181147478592698, "token_perplexity": 73.63766058317236, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.358194499500757, "token_perplexity": 160.53607401010768, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8997602150956525, "token_perplexity": 34.42473572192778, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0797127113417115, "token_perplexity": 71.90458647432683, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "ac266703745078d5fae728557a49eae60395ebeea2944678cc6b38e894eb7d35" }, { "step": 3000, "presented_tokens": 4608000, "components": { "bnc_spoken": { "bits_per_byte": 1.938364811716157, "token_perplexity": 94.14992874339121, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.821267946972846, "token_perplexity": 20.63591820167678, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.072292062852429, "token_perplexity": 127.7135980323179, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.162422504991228, "token_perplexity": 70.96939147715699, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.3147262735441014, "token_perplexity": 146.1900001490758, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8436817927091271, "token_perplexity": 31.010174928173555, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.035214678654062, "token_perplexity": 65.61890560904263, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "7d3b20619b6ed0d057efe8c70243a93dd353f1e40e23ff455c77b6a327d4a6ab" }, { "step": 3500, "presented_tokens": 5376000, "components": { "bnc_spoken": { "bits_per_byte": 1.9354447586839338, "token_perplexity": 93.50751833868902, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9847274599282732, "token_perplexity": 27.077701693923967, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0477278735146576, "token_perplexity": 120.57869478026059, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1399743242157294, "token_perplexity": 67.89769800115918, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2802210496875093, "token_perplexity": 135.72055085234064, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8024773305442798, "token_perplexity": 28.719083344772848, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0361903133458257, "token_perplexity": 65.75064605470024, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "cfdfd861a6df22582a85cf2b74ab31157355fbd45912a0c9439476745089548f" }, { "step": 4000, "presented_tokens": 6144000, "components": { "bnc_spoken": { "bits_per_byte": 1.9023205317717145, "token_perplexity": 86.51998181886276, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.852508492065582, "token_perplexity": 21.735707557462455, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0254216807938032, "token_perplexity": 114.44557362666895, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1117655659522736, "token_perplexity": 64.22555487949097, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.243122837538664, "token_perplexity": 125.29922654100643, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7951656889832732, "token_perplexity": 28.33058824988747, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9957556563025556, "token_perplexity": 60.506241283890745, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "16fd8e306f12b3fed6c0c1dc6fff0f4e21e8f4351b5f6e690ab6dd7a3f45e5b5" }, { "step": 4500, "presented_tokens": 6912000, "components": { "bnc_spoken": { "bits_per_byte": 1.8934584281666782, "token_perplexity": 84.74073306697798, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9720776584894173, "token_perplexity": 26.514347652368322, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.013846318456996, "token_perplexity": 111.38687717032886, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.098884187160017, "token_perplexity": 62.61540326346119, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2287748976039974, "token_perplexity": 121.48678689892955, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7829959018120676, "token_perplexity": 27.695579782536498, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.00158507208327, "token_perplexity": 61.23569435814019, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "ca0078c32b68378a3994be6e2d5a07e0a36f53be5880b4ce025bb6b2e0382985" }, { "step": 5000, "presented_tokens": 7680000, "components": { "bnc_spoken": { "bits_per_byte": 1.884775742701905, "token_perplexity": 83.03299700455273, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.950396895446842, "token_perplexity": 25.575926996695667, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0023380178010486, "token_perplexity": 108.42694376833076, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.085508436245959, "token_perplexity": 60.98616265058427, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.201651257559846, "token_perplexity": 114.59372641895919, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7621016334234507, "token_perplexity": 26.638354082660282, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9848008804473614, "token_perplexity": 59.158861639265496, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "170035bd026f550a67639d7de348f9aafea78b3c5efba1b665fab79ab6c75c64" }, { "step": 5500, "presented_tokens": 8448000, "components": { "bnc_spoken": { "bits_per_byte": 1.8702286113731732, "token_perplexity": 80.2486059311208, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.859681312729398, "token_perplexity": 21.996381965474054, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9897579312634095, "token_perplexity": 105.28125722379181, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0747991451040106, "token_perplexity": 59.71232330518912, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1803591915106484, "token_perplexity": 109.45782000327539, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.750004514222722, "token_perplexity": 26.04480099150145, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9601690091213197, "token_perplexity": 56.23783627928567, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "6234dac54acfc94ce6b445839b4aad06de063ced3c1c8476d174d1701e5917a0" }, { "step": 6000, "presented_tokens": 9216000, "components": { "bnc_spoken": { "bits_per_byte": 1.8717036092611181, "token_perplexity": 80.52662066710774, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9178310228910198, "token_perplexity": 24.228401057501152, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9733011058878078, "token_perplexity": 101.30355050824429, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0589329983473172, "token_perplexity": 57.87383649817227, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1549564837670805, "token_perplexity": 103.63063149041821, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7719916750091895, "token_perplexity": 27.133651750992936, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9615123452807532, "token_perplexity": 56.393354194255544, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "61eb32301dddf25589d03842f0d8015e2e3bead81904d5fa8aace4b886fb9cd4" }, { "step": 6500, "presented_tokens": 9984000, "components": { "bnc_spoken": { "bits_per_byte": 1.8517428152602422, "token_perplexity": 76.84463843993659, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8775669968226194, "token_perplexity": 22.660081077356388, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9681512409252047, "token_perplexity": 100.0899436629992, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0410859138527435, "token_perplexity": 55.87336913357035, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1354766053157896, "token_perplexity": 99.37313701746544, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7419306509277934, "token_perplexity": 25.656029032635267, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9405223052436016, "token_perplexity": 54.01173509792094, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "3dd829d61f79ce2bca0345be88f07239d71c074e1c8e01f88139ab52ef856cb7" }, { "step": 7000, "presented_tokens": 10752000, "components": { "bnc_spoken": { "bits_per_byte": 1.8552485662168434, "token_perplexity": 77.47890017172757, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.874738345535069, "token_perplexity": 22.55379809819469, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9548810857995642, "token_perplexity": 97.02930453599583, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0395812316381168, "token_perplexity": 55.70790494196116, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.125844487941331, "token_perplexity": 97.33303624678868, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7185218829710902, "token_perplexity": 24.561347026334275, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9328323254578061, "token_perplexity": 53.164600576832456, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "1fa5610632a32d7e4eefd5f1f1f06a48beff978619f8df1aed8c8fe5dcfbcbb3" }, { "step": 7500, "presented_tokens": 11520000, "components": { "bnc_spoken": { "bits_per_byte": 1.8319475182322713, "token_perplexity": 73.35947136233119, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8074214960803947, "token_perplexity": 20.166437357038692, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9540560645227498, "token_perplexity": 96.84214116490024, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.026882145271554, "token_perplexity": 54.33081104545871, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1255350218782607, "token_perplexity": 97.26818982416204, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7043853978555636, "token_perplexity": 23.92302137088916, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9149572718135, "token_perplexity": 51.246451825327014, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "d0459c4de46ea060ae1faf7293f84367f8609400c09481ba5ad37f17f9edd347" }, { "step": 8000, "presented_tokens": 12288000, "components": { "bnc_spoken": { "bits_per_byte": 1.8307127772588918, "token_perplexity": 73.14739549225078, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8608831778898058, "token_perplexity": 22.040364896638774, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9476023294365472, "token_perplexity": 95.39045776006691, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.023430413970683, "token_perplexity": 53.962423560288, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1083537402341666, "token_perplexity": 93.73494330101434, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6954414298089513, "token_perplexity": 23.52775693028927, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.915799806763349, "token_perplexity": 51.33528909517959, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "6f208781bd4fc3cb65097a8673130d24a35c159edd4c7127f4085d7c1d649a84" }, { "step": 8500, "presented_tokens": 13056000, "components": { "bnc_spoken": { "bits_per_byte": 1.8284273051179587, "token_perplexity": 72.75646513525801, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8714784637772701, "token_perplexity": 22.43193032357196, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.940170411800059, "token_perplexity": 93.74568290162047, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0125746733150587, "token_perplexity": 52.82004229526491, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.094159507543582, "token_perplexity": 90.9129889142289, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7007380401057755, "token_perplexity": 23.76103662776235, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9119697937909839, "token_perplexity": 50.93268796639216, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "fbf4b14427f6577d5bf58258c9f2a543f1403674459ff2d920da3855ebfacb95" }, { "step": 9000, "presented_tokens": 13824000, "components": { "bnc_spoken": { "bits_per_byte": 1.8228850257624865, "token_perplexity": 71.81711187444697, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8734743099115319, "token_perplexity": 22.506464860294557, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9301206117713283, "token_perplexity": 91.56655114963591, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.001176303158185, "token_perplexity": 51.646576956896574, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.087762847479565, "token_perplexity": 89.66919410783304, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6976493473773646, "token_perplexity": 23.62472103129805, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.905873580367773, "token_perplexity": 50.29837208502303, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "f9446584a1dea2814db1e9c291eedb36aa137bdcaf252656236ceba7258d4366" }, { "step": 9500, "presented_tokens": 14592000, "components": { "bnc_spoken": { "bits_per_byte": 1.8141512917805722, "token_perplexity": 70.36139653879117, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8650258566894788, "token_perplexity": 22.192643667939983, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9262807549078858, "token_perplexity": 90.74738128390602, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 1.996785033773346, "token_perplexity": 51.20148298850049, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.081809758322328, "token_perplexity": 88.52694150477272, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6805942351030947, "token_perplexity": 22.88597508898796, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8980295672498348, "token_perplexity": 49.49380415793831, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "2bca8b2e4ee8882f6941938cba4290a562b7a9949f5b96e8fac611f447b3d244" }, { "step": 10000, "presented_tokens": 15360000, "components": { "bnc_spoken": { "bits_per_byte": 1.8123344134096406, "token_perplexity": 70.06229179973155, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8721873280918238, "token_perplexity": 22.45837443361484, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9211110625014063, "token_perplexity": 89.65607599913025, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 1.9898252652255788, "token_perplexity": 50.50389316988436, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0708566290977464, "token_perplexity": 86.46318217520088, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6778211008765995, "token_perplexity": 22.768058937873118, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8942249323028821, "token_perplexity": 49.10820695645153, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "d8eef39288859f046dd8cca03a8bc47b8700db0ce0884160d1c6e00e80fff187" }, { "step": 10500, "presented_tokens": 16128000, "components": { "bnc_spoken": { "bits_per_byte": 1.8076036664556911, "token_perplexity": 69.28944153369679, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8664705066140397, "token_perplexity": 22.245993921118234, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9160817110221402, "token_perplexity": 88.60699179963284, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 1.9904807409789251, "token_perplexity": 50.56918521931264, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0626302989385246, "token_perplexity": 84.9448964797784, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6820549751033698, "token_perplexity": 22.948332423057458, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8909089001154018, "token_perplexity": 48.7745803293892, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "993c1bb9f039c45c0bbf5daf2429bba2389b0b58495da661010159f19f208321" }, { "step": 11000, "presented_tokens": 16896000, "components": { "bnc_spoken": { "bits_per_byte": 1.8057511701060085, "token_perplexity": 68.9891318831831, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8645367132701065, "token_perplexity": 22.174608837438505, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.910642846261309, "token_perplexity": 87.48629740887844, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 1.9868800899338945, "token_perplexity": 50.21156256345387, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0606106358816754, "token_perplexity": 84.57623351459014, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6733084011110284, "token_perplexity": 22.577472238784118, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8870152897838697, "token_perplexity": 48.385735430537586, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "bb15957c456ec46b845f287b3dd0fcef9ff9ecfbba325552edbe996d6b544dee" }, { "step": 11500, "presented_tokens": 17664000, "components": { "bnc_spoken": { "bits_per_byte": 1.801314300075362, "token_perplexity": 68.27514859220364, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8585324115363173, "token_perplexity": 21.954419336960225, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.908617353325241, "token_perplexity": 87.07257043916512, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 1.9827463434944803, "token_perplexity": 49.80410937113403, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0584431011523696, "token_perplexity": 84.18235834304707, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6718259943122022, "token_perplexity": 22.515213912594206, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8837155839604611, "token_perplexity": 48.05863001119832, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "a0afb0459053a717d103ab434f0f72c197b867eedf949e50e5deb17968eded11" }, { "step": 12000, "presented_tokens": 18432000, "components": { "bnc_spoken": { "bits_per_byte": 1.800188011382718, "token_perplexity": 68.09508464131812, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8734468316314874, "token_perplexity": 22.50543700909506, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9065041896913029, "token_perplexity": 86.64302111637764, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 1.9822473021267646, "token_perplexity": 49.75514430092777, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0522812962501806, "token_perplexity": 83.07265180977637, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6630559276225585, "token_perplexity": 22.150383865223162, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8826301916079489, "token_perplexity": 47.95151729446785, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "54e3a64ad89bf8af1579aca37321cb26082afda0bbb95b197afa1f75766a4bdf" } ], "source_commit": "d71b74bb167a7243485b508f3d5415bf4f1757eb", "protocol": { "steps": 12000, "batch": 16, "sequence": 96, "warmup": 16, "learning_rate": 0.002, "final_learning_rate": 0.0002, "lr_warmup_updates": 100, "weight_decay": 0.01, "eval_tokens": 49152, "threads": 4, "tokenizer_sha256": "46a2ef1bf8dce00e801c5ac188b13ec5c30c7f1ab7e8184031dca3ee057a7242", "train_cache_sha256": "4d205d6d35c87c38749ca3dc858fe96b28e3d7f4b8d7e2eb782e06909c8f15cd", "validation_cache_sha256": "ea3bd0f2625e78723baf6300dca2ffdae91deebdcb21ae748ffd0cc4ba2ef859", "graph_sha256": "a2f35369a5ed4981b8ef75c92df285f7df576a020582f119a1372bfbedbaebee", "validation_panel_sha256": "9a0c030150fee1821246dd54f045fc36c437bab3061e3010740d2942900de3ae", "evaluation_unit": "block", "cache_identity": "SHA-256 of verified mmap manifest" } }, { "scale": "10m", "seed": 43, "variant": "flm", "complete": true, "parameters": 600003, "validation": [ { "step": 0, "presented_tokens": 0, "components": { "bnc_spoken": { "bits_per_byte": 3.653515902165921, "token_perplexity": 5252.225987162989, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 5.073145501114346, "token_perplexity": 4590.78772996143, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 3.663569060637094, "token_perplexity": 5291.514275372805, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 4.349295118839944, "token_perplexity": 5285.327207653864, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 3.9872834982124745, "token_perplexity": 5360.771951201961, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 4.508799105867208, "token_perplexity": 4441.549044437215, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 4.14534326597165, "token_perplexity": 5022.740146813829, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "c8af6de8eca7bfe34a4f4cb5c859229ab289193197dd764d2a88086e09ecd8c6" }, { "step": 500, "presented_tokens": 768000, "components": { "bnc_spoken": { "bits_per_byte": 2.2619206700230805, "token_perplexity": 201.04538416301915, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.1764361946882618, "token_perplexity": 37.23836419306761, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.419172357778494, "token_perplexity": 287.6070256142509, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.4647218678183864, "token_perplexity": 128.77885719816283, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.731714000530603, "token_perplexity": 358.8542002483589, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.4144300330581236, "token_perplexity": 89.79019350778563, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.417703076200825, "token_perplexity": 144.0497857688687, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "030620480045849d8275172d4bc1b9e6384b5bacae60d507da68f604a0bb8e38" }, { "step": 1000, "presented_tokens": 1536000, "components": { "bnc_spoken": { "bits_per_byte": 2.1348704405516745, "token_perplexity": 149.25193318396225, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.0682965190511635, "token_perplexity": 31.112394853195294, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.2907721011387783, "token_perplexity": 212.95899079480938, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.347929940384776, "token_perplexity": 102.2983563839866, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.572972359420865, "token_perplexity": 254.94712508743464, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.198277907943822, "token_perplexity": 60.0296575652312, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.2755947003524155, "token_perplexity": 107.55708033003154, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "26972372f43a9a4b3d8902a7f16b831bbb44a4bf3afa0ea05bf0244118b84e2a" }, { "step": 1500, "presented_tokens": 2304000, "components": { "bnc_spoken": { "bits_per_byte": 2.0603544323160765, "token_perplexity": 125.32593784330662, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.020057898043596, "token_perplexity": 28.715338618696947, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.2229003762349615, "token_perplexity": 181.68224415936052, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.286644857369763, "token_perplexity": 90.65829863993223, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.48037419734153, "token_perplexity": 208.85500679029306, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.1574818511324247, "token_perplexity": 55.636860591342966, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.2099298195665202, "token_perplexity": 93.97530496444968, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "971a8d25f4842206a0d60d1af59bcddf652b011765c27451b58e23cac832c141" }, { "step": 2000, "presented_tokens": 3072000, "components": { "bnc_spoken": { "bits_per_byte": 2.023261790122196, "token_perplexity": 114.88672485657784, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.969179028868226, "token_perplexity": 26.386917769168047, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.1819660373453074, "token_perplexity": 165.08500373735353, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.2466650000864496, "token_perplexity": 83.7884419734175, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.4206945077743036, "token_perplexity": 183.665530899221, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.0197819797497854, "token_perplexity": 43.049289296491416, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.1506518115599076, "token_perplexity": 83.19374014278229, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "17d257a3e58eefa7ad718481a0ac2c65c6331169ad7fd6dd63fa0a79e962ccaa" }, { "step": 2500, "presented_tokens": 3840000, "components": { "bnc_spoken": { "bits_per_byte": 1.9963309629856345, "token_perplexity": 107.85652464819772, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9803876982389204, "token_perplexity": 26.88309593067835, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.154310371295823, "token_perplexity": 154.738700593689, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.230049468183022, "token_perplexity": 81.08881093356476, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.3739014642354395, "token_perplexity": 166.0592419630154, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.0042011407253564, "token_perplexity": 41.81781891114897, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.128617279873681, "token_perplexity": 79.5093741717985, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "fe9ce6fe8e88d2295823c2f4da9a955d917f3d6bf3489f07aa2df7378eb23b9f" }, { "step": 3000, "presented_tokens": 4608000, "components": { "bnc_spoken": { "bits_per_byte": 1.9795530932816474, "token_perplexity": 103.6959187540208, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9191285201763524, "token_perplexity": 24.280706021617107, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.1357747261378695, "token_perplexity": 148.16979438351257, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1977600517005467, "token_perplexity": 76.08878212553512, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.3352261618263044, "token_perplexity": 152.78854891925994, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.025986213717826, "token_perplexity": 43.54969278899166, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.105033878306736, "token_perplexity": 75.74660946996518, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "f4d37933ff9d2b033c5a369ee0222b2264988e59931ae88da974938b2addda12" }, { "step": 3500, "presented_tokens": 5376000, "components": { "bnc_spoken": { "bits_per_byte": 1.9629878605911375, "token_perplexity": 99.74552757560848, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9387012355086848, "token_perplexity": 25.083564211972845, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.117606215010354, "token_perplexity": 142.0017049149132, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1922620925990106, "token_perplexity": 75.26867738759913, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.320274539920958, "token_perplexity": 147.94723406783783, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.0085257549204902, "token_perplexity": 42.1560498995809, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0949490845936976, "token_perplexity": 74.1924217034857, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "8304c63fde14bbb68dfb7fef9353ffb2156248b197973b1114c2d31b59f2b4e0" }, { "step": 4000, "presented_tokens": 6144000, "components": { "bnc_spoken": { "bits_per_byte": 1.9519096364720963, "token_perplexity": 97.1879838110268, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.929833285475671, "token_perplexity": 24.716569989797424, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.1032319278587304, "token_perplexity": 137.30419751946917, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.174813095102385, "token_perplexity": 72.72398037516412, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.299127372836395, "token_perplexity": 141.36058164944677, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.0142940840599577, "token_perplexity": 42.611456661610674, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0833114063450076, "token_perplexity": 72.43850672026124, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "8f71e6bf08128419d2d29665b164c0c513c88f39d8ebab5041861724f33072ac" }, { "step": 4500, "presented_tokens": 6912000, "components": { "bnc_spoken": { "bits_per_byte": 1.935397443079679, "token_perplexity": 93.49714509579354, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9202390768467477, "token_perplexity": 24.32556467592981, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0970376853993415, "token_perplexity": 135.32813829807526, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.163965729983401, "token_perplexity": 71.18559324594507, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.283912517956115, "token_perplexity": 136.8038031216261, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.9118277029394741, "token_perplexity": 35.207321627583795, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.057840027709394, "token_perplexity": 68.74304664107352, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "81d959b2d1a11235f244c63bf627c4a51abe3deb718ee9609546360ad83512d9" }, { "step": 5000, "presented_tokens": 7680000, "components": { "bnc_spoken": { "bits_per_byte": 1.9352012868090682, "token_perplexity": 93.45415302210476, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9140935666312144, "token_perplexity": 24.07836486872954, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0880876112979547, "token_perplexity": 132.5230513403348, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1605583125762067, "token_perplexity": 70.70909852800851, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2636900195535974, "token_perplexity": 130.97381215745978, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8713558771490912, "token_perplexity": 32.65066527929062, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0450890888943323, "token_perplexity": 66.96452684907715, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "0bf5302f972ce265b212eda165b9fcfbcdb330a63cc3a1453591b18c73c34c2a" }, { "step": 5500, "presented_tokens": 8448000, "components": { "bnc_spoken": { "bits_per_byte": 1.9194451397501964, "token_perplexity": 90.06462779527659, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8930220001615445, "token_perplexity": 23.249689508976484, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0772037406171338, "token_perplexity": 129.19011164880564, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.144237306045449, "token_perplexity": 68.47061622213924, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2448162725081207, "token_perplexity": 125.75701675492208, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8943613234704464, "token_perplexity": 34.08026885052179, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.034760350038947, "token_perplexity": 65.55764749870804, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "7532f6c06f1bf78194d63688941e107ae2d9b44a485a3543317284c0f1977b3b" }, { "step": 6000, "presented_tokens": 9216000, "components": { "bnc_spoken": { "bits_per_byte": 1.9154388170122492, "token_perplexity": 89.22255487527053, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9038539630344498, "token_perplexity": 23.672049596934652, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0706391980398093, "token_perplexity": 127.22053008975219, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1380361392312204, "token_perplexity": 67.6388056549734, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2436353524332007, "token_perplexity": 125.43759974847158, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8671514382900958, "token_perplexity": 32.39595015269326, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.028906436632981, "token_perplexity": 64.77344825210294, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "8aebcba8c279eee920d3628fc72f06086d57738e03af70cd8c88a6deac335a64" }, { "step": 6500, "presented_tokens": 9984000, "components": { "bnc_spoken": { "bits_per_byte": 1.904589458028505, "token_perplexity": 86.98149089063568, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9040505898019393, "token_perplexity": 23.67978695689919, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0622767841608334, "token_perplexity": 124.75495584680023, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.134972349406311, "token_perplexity": 67.23157312405206, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2356011048667264, "token_perplexity": 123.28591491863814, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.890342156337919, "token_perplexity": 33.82607292549016, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0268937784412904, "token_perplexity": 64.50600217972315, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "ce1b36beba738f9a485f7609ef4935c69443ccdcd0ce4a6aba4cf76e89a5b8f9" }, { "step": 7000, "presented_tokens": 10752000, "components": { "bnc_spoken": { "bits_per_byte": 1.898368492371353, "token_perplexity": 85.72195995322184, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8750921159229277, "token_perplexity": 22.567063248804967, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.054317330924316, "token_perplexity": 122.45258966978464, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.122197543954922, "token_perplexity": 65.55982999804802, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2173487826282003, "token_perplexity": 118.53385987071043, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8800144880190894, "token_perplexity": 33.18155155593615, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.013510950004016, "token_perplexity": 62.755530893304766, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "819e9e0b1f4392179f4f144e6ccbef04491fba0de27a4da8b1a92fd4f0eaa2c0" }, { "step": 7500, "presented_tokens": 11520000, "components": { "bnc_spoken": { "bits_per_byte": 1.896514268189041, "token_perplexity": 85.3500838079408, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9010394459313242, "token_perplexity": 23.56157365838467, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0483139419384266, "token_perplexity": 120.74419126045271, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1177687986977447, "token_perplexity": 64.99002980926485, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2143446351990037, "token_perplexity": 117.76946634801676, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8577421970434211, "token_perplexity": 31.83309176612573, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0109199679708483, "token_perplexity": 62.42216003569433, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "f3213bfba41a00bc8b8164d8969f197b08a953775a7cedc9407ee5215cc52e59" }, { "step": 8000, "presented_tokens": 12288000, "components": { "bnc_spoken": { "bits_per_byte": 1.893164269229314, "token_perplexity": 84.68230625105838, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8775236304730702, "token_perplexity": 22.658447867439357, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0431270326378423, "token_perplexity": 119.28734533909352, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.108834969178444, "token_perplexity": 63.855634338580245, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.203064432866505, "token_perplexity": 114.94300772268531, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8562657619134593, "token_perplexity": 31.74566380888641, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.002510167275466, "token_perplexity": 61.352260229166205, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "8232251b8d4e2225a89cfdb079cd3d38ea859ea36d5041684bf5e99883739d79" }, { "step": 8500, "presented_tokens": 13056000, "components": { "bnc_spoken": { "bits_per_byte": 1.8866749540850527, "token_perplexity": 83.4035745224345, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.858386934333635, "token_perplexity": 21.949111620339323, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0406310987354126, "token_perplexity": 118.59259038253492, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.102949096950952, "token_perplexity": 63.119102513727235, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.19876972546295, "token_perplexity": 113.88481019980773, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8404836510497866, "token_perplexity": 30.82598615991513, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9941505306637337, "token_perplexity": 60.30691700710287, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "8f039f2288f8c806843dc1b86438d7acf05e29b6cdecdc5aadde66bbaea7afe8" }, { "step": 9000, "presented_tokens": 13824000, "components": { "bnc_spoken": { "bits_per_byte": 1.8854030452471935, "token_perplexity": 83.15521492972094, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8741563952586864, "token_perplexity": 22.531993961953468, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0352300622733934, "token_perplexity": 117.10300708682391, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0997434644882684, "token_perplexity": 62.721543633501, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1917482068852454, "token_perplexity": 112.17568106489556, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8329933603950792, "token_perplexity": 30.398872337251493, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9921394913210513, "token_perplexity": 60.05811286982324, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "9d41a462619cbdcf45fb91898e4683f2621f2e0f36f6e7a1bc8ac2e456434267" }, { "step": 9500, "presented_tokens": 14592000, "components": { "bnc_spoken": { "bits_per_byte": 1.8826826978907256, "token_perplexity": 82.62650526263158, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.857509720585989, "token_perplexity": 21.91713377817467, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.034015403708749, "token_perplexity": 116.77059475675621, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.093500447395846, "token_perplexity": 61.95446449987358, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1850241701286603, "token_perplexity": 110.56301075294432, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8499618024753117, "token_perplexity": 31.37506447673465, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9895397215955797, "token_perplexity": 59.73799208906213, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "7da8983972d722b956ea68c03bdbe7a30a3a63e59d032e850681b177d1af38c7" }, { "step": 10000, "presented_tokens": 15360000, "components": { "bnc_spoken": { "bits_per_byte": 1.8774181324780126, "token_perplexity": 81.6128460937894, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.860517702980687, "token_perplexity": 22.02698083941793, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.031564503381681, "token_perplexity": 116.10273356901357, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.091800646268014, "token_perplexity": 61.74723974893899, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1839294438770533, "token_perplexity": 110.30265871364716, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.824968510382307, "token_perplexity": 29.947841047922147, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9842419874753587, "token_perplexity": 59.09093090488798, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "c34ac3788d3a5a1b4b36320a692f805079da4b6abaaa008c74a62098875e6625" }, { "step": 10500, "presented_tokens": 16128000, "components": { "bnc_spoken": { "bits_per_byte": 1.877299750124693, "token_perplexity": 81.59019584713766, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8712904697105917, "token_perplexity": 22.424922449696133, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0288614191712617, "token_perplexity": 115.37058190180143, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0920407131700203, "token_perplexity": 61.776464500193754, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1778038658783063, "token_perplexity": 108.8571221872861, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.828281125693951, "token_perplexity": 30.133207587485586, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9846273466291962, "token_perplexity": 59.13776108879411, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "6a4ea319b66ed1ffa748a2e08754635bf9e72933960a58f12a50b16a22570b89" }, { "step": 11000, "presented_tokens": 16896000, "components": { "bnc_spoken": { "bits_per_byte": 1.8749004722022926, "token_perplexity": 81.13249128643545, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8560709341289918, "token_perplexity": 21.86478527522189, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.026922578181933, "token_perplexity": 114.8482767133893, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.08848081903797, "token_perplexity": 61.34451239986626, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.174621699350453, "token_perplexity": 108.11367533807837, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8257774345265263, "token_perplexity": 29.99300113473352, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9802499902362292, "token_perplexity": 58.607986006460735, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "00ad739ea6823d9f651f532d80e30b32b745f692e4261232e504d8f634c3546b" }, { "step": 11500, "presented_tokens": 17664000, "components": { "bnc_spoken": { "bits_per_byte": 1.8740445126560281, "token_perplexity": 80.96982407106835, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8593589811518534, "token_perplexity": 21.984600993762346, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0226480305578147, "token_perplexity": 113.70509264765202, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0856137083215716, "token_perplexity": 60.998818397572315, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.172195090150493, "token_perplexity": 107.55016275563298, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8214172553035144, "token_perplexity": 29.750387047630248, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9781789708766515, "token_perplexity": 58.3589947459688, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "f9da064abfab6a51182fd49486e69e01f158e0e798cfd39b7e4386fe0cc44392" }, { "step": 12000, "presented_tokens": 18432000, "components": { "bnc_spoken": { "bits_per_byte": 1.8725905125023667, "token_perplexity": 80.69425195170169, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8493194919202571, "token_perplexity": 21.620807240955475, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0201218795417764, "token_perplexity": 113.03485613252433, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0862010420015755, "token_perplexity": 61.06947552358171, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1705329641861146, "token_perplexity": 107.1658760213138, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.811055246888883, "token_perplexity": 29.18165700003002, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.974309237359898, "token_perplexity": 57.89658190592817, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "4dbcc30e87beb92d2898bca9a212ca0ee569e15c9db5f7ed4ad62df8bce25c1e" } ], "source_commit": "2855cc224ba86a84db7e501ad15bb12afc036381", "protocol": { "steps": 12000, "batch": 16, "sequence": 96, "warmup": 16, "learning_rate": 0.002, "final_learning_rate": 0.0002, "lr_warmup_updates": 100, "weight_decay": 0.01, "eval_tokens": 49152, "threads": 4, "tokenizer_sha256": "46a2ef1bf8dce00e801c5ac188b13ec5c30c7f1ab7e8184031dca3ee057a7242", "train_cache_sha256": "4d205d6d35c87c38749ca3dc858fe96b28e3d7f4b8d7e2eb782e06909c8f15cd", "validation_cache_sha256": "ea3bd0f2625e78723baf6300dca2ffdae91deebdcb21ae748ffd0cc4ba2ef859", "graph_sha256": "a2f35369a5ed4981b8ef75c92df285f7df576a020582f119a1372bfbedbaebee", "validation_panel_sha256": "9a0c030150fee1821246dd54f045fc36c437bab3061e3010740d2942900de3ae", "evaluation_unit": "block", "cache_identity": "SHA-256 of verified mmap manifest" } }, { "scale": "10m", "seed": 43, "variant": "gru", "complete": true, "parameters": 595408, "validation": [ { "step": 0, "presented_tokens": 0, "components": { "bnc_spoken": { "bits_per_byte": 3.5452520671982803, "token_perplexity": 4074.73420074953, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 5.011258447882946, "token_perplexity": 4142.05660518719, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 3.5530831860636027, "token_perplexity": 4085.8675966965293, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 4.219522609329652, "token_perplexity": 4092.459266471993, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 3.861212057914252, "token_perplexity": 4086.158511624914, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 4.461841317730949, "token_perplexity": 4069.550915179262, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 4.045619225940278, "token_perplexity": 4091.7360537654986, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "868ff969a670eb257bf4c1c757e80ef3a5f8edd37000c5ad0cc76aae3371783b" }, { "step": 500, "presented_tokens": 768000, "components": { "bnc_spoken": { "bits_per_byte": 2.2664883270537186, "token_perplexity": 203.21011043181213, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.209755727912891, "token_perplexity": 39.35874395074718, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.4255205112485143, "token_perplexity": 291.9117858119448, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.4704918575081876, "token_perplexity": 130.25181322645162, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.747507307853727, "token_perplexity": 371.26943880992764, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.5206419674833387, "token_perplexity": 109.43402762892102, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.4439853085185606, "token_perplexity": 152.04679000506275, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "1ecf1db8a3bd6627d916a22f1deb6892fd63df6f77754806ebb5793292a0c3d2" }, { "step": 1000, "presented_tokens": 1536000, "components": { "bnc_spoken": { "bits_per_byte": 2.1496170161966517, "token_perplexity": 154.50277157584608, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.0930023292859503, "token_perplexity": 32.41653101035674, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.2731817121762146, "token_perplexity": 204.3701447685498, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.34290699129625, "token_perplexity": 101.29054891524318, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.574414867405174, "token_perplexity": 255.74035763533786, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.218810308398435, "token_perplexity": 62.370064810596425, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.280939457203947, "token_perplexity": 108.74537006214035, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "85e4f298b2e5d58d59adf84350a41bd8f2ad2068c60021a7e23afa2aab625cc9" }, { "step": 1500, "presented_tokens": 2304000, "components": { "bnc_spoken": { "bits_per_byte": 2.061659811059385, "token_perplexity": 125.71011365081368, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.051667417970757, "token_perplexity": 30.264273923951883, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.2017958466346443, "token_perplexity": 172.92677986775078, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.2662838351879437, "token_perplexity": 87.09198261519127, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.4765812902836313, "token_perplexity": 207.15597300244323, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.112664081329671, "token_perplexity": 51.18065923216203, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.1997486409124245, "token_perplexity": 92.02885970415993, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "bd4d3091053aa8deff38e125fdee1cfda8a8420ca685dee29935927c09da9ea9" }, { "step": 2000, "presented_tokens": 3072000, "components": { "bnc_spoken": { "bits_per_byte": 2.0061772607110697, "token_perplexity": 110.37553496388024, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.005279003394234, "token_perplexity": 28.018589686261887, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.143061994644121, "token_perplexity": 150.7184140117771, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.2124311644532444, "token_perplexity": 78.32120474847137, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.400053130016624, "token_perplexity": 175.67994486196991, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.9327901565471415, "token_perplexity": 36.60928290056155, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.12267492219246, "token_perplexity": 78.54400263703104, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "611c6cc2331610148ca01361ea3ca1710840c400159753d6c9e142d22e95e799" }, { "step": 2500, "presented_tokens": 3840000, "components": { "bnc_spoken": { "bits_per_byte": 1.9772029962283362, "token_perplexity": 103.12609678996557, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.0038692653086527, "token_perplexity": 27.953017547317213, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.1020921013400136, "token_perplexity": 136.93842132020785, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.181114533996247, "token_perplexity": 73.63287903751987, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.3341493635024655, "token_perplexity": 152.43465006848515, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.916552902921662, "token_perplexity": 35.5185793174509, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0902403461844106, "token_perplexity": 73.47770956935584, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "9b32c2ca177d43f6f12fd8cca91893337cbf55efe749a26cff2db8b8ff3712d3" }, { "step": 3000, "presented_tokens": 4608000, "components": { "bnc_spoken": { "bits_per_byte": 1.946888604103496, "token_perplexity": 96.05051530507129, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9571988842873658, "token_perplexity": 25.866709214210587, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0800526813370595, "token_perplexity": 130.0543495057768, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1464661116618635, "token_perplexity": 68.7720757116874, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.279688721353697, "token_perplexity": 135.56504966338528, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8895808095554798, "token_perplexity": 33.778134873349515, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0548883679292866, "token_perplexity": 68.32718827431638, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "9ae7da758c56e405b3c54e80aa20ed4a2b949a1415a8ec2b851fde146a0e2536" }, { "step": 3500, "presented_tokens": 5376000, "components": { "bnc_spoken": { "bits_per_byte": 1.9249838903550747, "token_perplexity": 91.24189993752577, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9487279790949061, "token_perplexity": 25.505082340026267, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0485525048008046, "token_perplexity": 120.81162268220798, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.131935018641582, "token_perplexity": 66.83027767069005, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.258177676495533, "token_perplexity": 129.42819156359266, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8623458767640486, "token_perplexity": 32.107250197595505, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.033415297325719, "token_perplexity": 65.37662619301811, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "c3c646105e3b778076128860fc38a373a862338dc6d685d84f6d613207827e52" }, { "step": 4000, "presented_tokens": 6144000, "components": { "bnc_spoken": { "bits_per_byte": 1.9086268523607708, "token_perplexity": 87.80880984707498, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9480501512617565, "token_perplexity": 25.476364957656404, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0264383800958137, "token_perplexity": 114.71820780310455, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1057785768149824, "token_perplexity": 63.472104589801965, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2211057248514576, "token_perplexity": 119.49678485771342, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8358822722015733, "token_perplexity": 30.562899218121125, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.011369645415305, "token_perplexity": 62.479890887353406, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "cbc0259349c83614ef7a67bf09346dab3cdc76221cefc19e9992e8716d7950c6" }, { "step": 4500, "presented_tokens": 6912000, "components": { "bnc_spoken": { "bits_per_byte": 1.885094332001044, "token_perplexity": 83.09504558246961, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.931253561440744, "token_perplexity": 24.774983992282902, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0161174203521277, "token_perplexity": 111.98048183175297, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.090370203267924, "token_perplexity": 61.573389931383346, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.210181510551288, "token_perplexity": 116.7183176327034, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7779658757328398, "token_perplexity": 27.437293170852996, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9895584765351884, "token_perplexity": 59.74029534383811, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "39310ec5bbe526001ce82585e92edc3d693fbab0de5a322e04218ce6ba54c585" }, { "step": 5000, "presented_tokens": 7680000, "components": { "bnc_spoken": { "bits_per_byte": 1.8882083813144892, "token_perplexity": 83.70398600073058, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.921529119279972, "token_perplexity": 24.37777734941543, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.00444734599616, "token_perplexity": 108.96351297722737, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.080258977228081, "token_perplexity": 60.35839459450597, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1827225184468557, "token_perplexity": 110.01633351791007, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7461476120924126, "token_perplexity": 25.858354282364665, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9754105586128081, "token_perplexity": 58.027809746280525, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "a03d0e6b563e365eb0d9b5eff0a400139b73e2e3878157f48f00057834b498e5" }, { "step": 5500, "presented_tokens": 8448000, "components": { "bnc_spoken": { "bits_per_byte": 1.8712746166205827, "token_perplexity": 80.445662879703, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8911336408463761, "token_perplexity": 23.17683376810294, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9918048482173916, "token_perplexity": 105.78680699220072, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.062053757508382, "token_perplexity": 58.230926060387404, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.164933306376794, "token_perplexity": 105.88130108000077, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7516057464325416, "token_perplexity": 26.122600600201917, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9605061540588953, "token_perplexity": 56.27682716245859, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "214c876a8c446323f9a4cf87db116954a0f9e4c3f8b8856c4904a8fdfc8c2886" }, { "step": 6000, "presented_tokens": 9216000, "components": { "bnc_spoken": { "bits_per_byte": 1.865806240837311, "token_perplexity": 79.42079595630081, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9227984165587482, "token_perplexity": 24.42925977145187, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9831947234092457, "token_perplexity": 103.67650449172095, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0566688298789786, "token_perplexity": 57.61613287776971, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1584803352521225, "token_perplexity": 104.42005873296738, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7382804223371968, "token_perplexity": 25.482173682035377, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9581001836645775, "token_perplexity": 55.99916696750752, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "bf78292b5487474823ecee5bae249fb2caaa5b3e21b65a7c0352cb2724afdd4b" }, { "step": 6500, "presented_tokens": 9984000, "components": { "bnc_spoken": { "bits_per_byte": 1.855722342690328, "token_perplexity": 77.56501657790513, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9298042034834546, "token_perplexity": 24.715375327400878, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.977482625615729, "token_perplexity": 102.29977576754074, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0501472248420063, "token_perplexity": 56.88025030524204, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1462018488518146, "token_perplexity": 101.69511876795087, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.738117380378062, "token_perplexity": 25.47443576415117, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9528329338906985, "token_perplexity": 55.39607487802284, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "082aceddcc4a587e7f10415059497866ba07fdd06bd7f663ec3da967d8028127" }, { "step": 7000, "presented_tokens": 10752000, "components": { "bnc_spoken": { "bits_per_byte": 1.8462807780409025, "token_perplexity": 75.8667756105846, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8897780714734649, "token_perplexity": 23.124674673672658, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9679574012374537, "token_perplexity": 100.04454880787955, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0430386820801076, "token_perplexity": 56.08884064701436, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1291443095223586, "token_perplexity": 98.02718445088745, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.721943178256304, "token_perplexity": 24.718376640797235, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9372805844171028, "token_perplexity": 53.652990999621714, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "aaddcb01ec930631410bd77fb9b90dbc021f22a3c98f766411a9901bc9be2663" }, { "step": 7500, "presented_tokens": 11520000, "components": { "bnc_spoken": { "bits_per_byte": 1.8487576018861274, "token_perplexity": 76.30864725884335, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9129077441959927, "token_perplexity": 24.03095572795914, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9594724506990675, "token_perplexity": 98.07752370171033, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.034017967353794, "token_perplexity": 55.1003776390431, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1276478439071256, "token_perplexity": 97.71177762789395, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7110908930280895, "token_perplexity": 24.223709458643754, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.935916478968534, "token_perplexity": 53.50274586831918, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "f2fea9db1db69e573d7786d098f8e02fc66b928954b74bf3f1c7e8c0489c7353" }, { "step": 8000, "presented_tokens": 12288000, "components": { "bnc_spoken": { "bits_per_byte": 1.8396701420362425, "token_perplexity": 74.69990914741308, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9050690422900494, "token_perplexity": 23.71990405577109, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9528463357584973, "token_perplexity": 96.5683560992458, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.02610502909439, "token_perplexity": 54.24765423943576, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1142579643713777, "token_perplexity": 94.93440329250987, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7060457667861832, "token_perplexity": 23.997126217116953, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9275112688879847, "token_perplexity": 52.58621879067172, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "396b5e5e6a384f50b9d5fd00c88c088e725a5356eb50310fc466cfa5e0e291f7" }, { "step": 8500, "presented_tokens": 13056000, "components": { "bnc_spoken": { "bits_per_byte": 1.8337001736601635, "token_perplexity": 73.66155911163152, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8830033390157035, "token_perplexity": 22.865753034673613, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9495048479685786, "token_perplexity": 95.81612773760116, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0163530949598982, "token_perplexity": 53.214886345435296, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1091720280109767, "token_perplexity": 93.900271805523, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6980691300999267, "token_perplexity": 23.64320159717743, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9191180819325757, "token_perplexity": 51.68666974188303, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "11006f56721aaa76eeb06dede593bed7a2199b68d7840512a95d70f0baa17a1a" }, { "step": 9000, "presented_tokens": 13824000, "components": { "bnc_spoken": { "bits_per_byte": 1.830441687071749, "token_perplexity": 73.10091589425382, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9018400860650142, "token_perplexity": 23.592947953241286, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9450740477980275, "token_perplexity": 94.82770431704311, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0139776130297635, "token_perplexity": 52.966306008584866, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.099552681949786, "token_perplexity": 91.97505695063913, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6868779106660066, "token_perplexity": 23.15542717434762, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9164469768402839, "token_perplexity": 51.40363156740739, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "e5e4a8c5d0170c09ddff81b9ed0ff965710065afb5fe69e30a0c578446f252d0" }, { "step": 9500, "presented_tokens": 14592000, "components": { "bnc_spoken": { "bits_per_byte": 1.8274470742881295, "token_perplexity": 72.58943736655145, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8775215078798813, "token_perplexity": 22.658367931977025, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.941804155927399, "token_perplexity": 94.1048014406769, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.007589008263834, "token_perplexity": 52.30352137661741, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.093061368343163, "token_perplexity": 90.698241803069, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6930057858682288, "token_perplexity": 23.421253472112113, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9107515573599316, "token_perplexity": 50.80529313964452, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "dc020cf0462336d27062a1b8581e3b35058d1aa7ac5254eb5c56b0d6547576c5" }, { "step": 10000, "presented_tokens": 15360000, "components": { "bnc_spoken": { "bits_per_byte": 1.8234329401578846, "token_perplexity": 71.90943433287606, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8815331709037528, "token_perplexity": 22.80994903179764, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9392245841489573, "token_perplexity": 93.53840418859014, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.00601478254059, "token_perplexity": 52.14148125597018, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0903426172982527, "token_perplexity": 90.16875521387698, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6856316427730045, "token_perplexity": 23.101734578020995, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9081997209683377, "token_perplexity": 50.53947141574303, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "2bbc03118c0bc5dee97faf624c8784be2ef2541349aad441fdc5748a2569f922" }, { "step": 10500, "presented_tokens": 16128000, "components": { "bnc_spoken": { "bits_per_byte": 1.8227180440451283, "token_perplexity": 71.78899936853108, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8899109408760533, "token_perplexity": 23.129781979950565, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9360781171613812, "token_perplexity": 92.85214668662502, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0027726333807565, "token_perplexity": 51.809336301754456, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0877238044176503, "token_perplexity": 89.66165488968126, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6812850457968413, "token_perplexity": 22.91544386761916, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9069639514360468, "token_perplexity": 50.41124309095138, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "144a6012c27aa06c53e9d8f35bea2236ba2af825b5842468c5e7f931cbfe4bf4" }, { "step": 11000, "presented_tokens": 16896000, "components": { "bnc_spoken": { "bits_per_byte": 1.8191989780528002, "token_perplexity": 71.19909373761789, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.879453941822791, "token_perplexity": 22.73125898684594, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.934215357779914, "token_perplexity": 92.44824617093633, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.001122561942662, "token_perplexity": 51.641106491712804, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.08157446291951, "token_perplexity": 88.48209420441356, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6801588722444694, "token_perplexity": 22.86742273954664, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9030250737370917, "token_perplexity": 50.004696707480484, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "c069ada0174de08ef26ba85331124a3ff4b9166fd7ddd8f4eea6f4ada4873e87" }, { "step": 11500, "presented_tokens": 17664000, "components": { "bnc_spoken": { "bits_per_byte": 1.8185744376565016, "token_perplexity": 71.09490883916556, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8826740703463738, "token_perplexity": 22.853242945064245, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9310788124732692, "token_perplexity": 91.77211766423474, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.001661211103133, "token_perplexity": 51.69596327346384, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.079981052605716, "token_perplexity": 88.17898751618391, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6774068138492948, "token_perplexity": 22.75049531982542, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9021373935885557, "token_perplexity": 49.91352954780215, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "1059c4c3cac6cf5a620e35c75580baeff1ab6e4bb7b774742178227ba0f8346d" }, { "step": 12000, "presented_tokens": 18432000, "components": { "bnc_spoken": { "bits_per_byte": 1.8163750266742351, "token_perplexity": 70.72921868510382, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8771069981542907, "token_perplexity": 22.642763172040254, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9282333721185563, "token_perplexity": 91.16301989497484, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 1.9995490289045539, "token_perplexity": 51.48118886884189, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0764775369677246, "token_perplexity": 87.51617722043548, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6753768704120098, "token_perplexity": 22.664631903128534, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8991734568320775, "token_perplexity": 49.61032751460638, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "f40224eb6070e4dd115714a3dbfc03a8c17fb232c0c773b1df17bdfed63656b8" } ], "source_commit": "6188aaf480a001e304b906819a4d8d152a34be72", "protocol": { "steps": 12000, "batch": 16, "sequence": 96, "warmup": 16, "learning_rate": 0.002, "final_learning_rate": 0.0002, "lr_warmup_updates": 100, "weight_decay": 0.01, "eval_tokens": 49152, "threads": 4, "tokenizer_sha256": "46a2ef1bf8dce00e801c5ac188b13ec5c30c7f1ab7e8184031dca3ee057a7242", "train_cache_sha256": "4d205d6d35c87c38749ca3dc858fe96b28e3d7f4b8d7e2eb782e06909c8f15cd", "validation_cache_sha256": "ea3bd0f2625e78723baf6300dca2ffdae91deebdcb21ae748ffd0cc4ba2ef859", "graph_sha256": "a2f35369a5ed4981b8ef75c92df285f7df576a020582f119a1372bfbedbaebee", "validation_panel_sha256": "9a0c030150fee1821246dd54f045fc36c437bab3061e3010740d2942900de3ae", "evaluation_unit": "block", "cache_identity": "SHA-256 of verified mmap manifest" } }, { "scale": "10m", "seed": 43, "variant": "transformer", "complete": true, "parameters": 607468, "validation": [ { "step": 0, "presented_tokens": 0, "components": { "bnc_spoken": { "bits_per_byte": 3.620318122014462, "token_perplexity": 4858.904843572235, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 5.117717416353461, "token_perplexity": 4943.790040721126, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 3.597350530048216, "token_perplexity": 4531.862787677862, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 4.309159989521228, "token_perplexity": 4883.323707516218, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 3.914240346163621, "token_perplexity": 4580.485559225271, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 4.577161867072917, "token_perplexity": 5044.73853515854, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 4.123631780135422, "token_perplexity": 4803.489102471148, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "f0e6f05cccf23ee01e5594781f4043e7fdf56b124052192b1fd286104c207390" }, { "step": 500, "presented_tokens": 768000, "components": { "bnc_spoken": { "bits_per_byte": 2.25719979912212, "token_perplexity": 198.832276445051, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.186879949387612, "token_perplexity": 37.89039075547345, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.392171761440284, "token_perplexity": 269.9955538284358, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.4526224899092504, "token_perplexity": 125.74400535734623, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.7446341463900294, "token_perplexity": 368.9792897327104, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.3758710501983846, "token_perplexity": 83.56711493365924, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.407589062565208, "token_perplexity": 141.08566139781522, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "f13c3cf6e6f1cedc9f091245ddd417130cdd0879f069c21340efb17d12c21f1d" }, { "step": 1000, "presented_tokens": 1536000, "components": { "bnc_spoken": { "bits_per_byte": 2.13187403082212, "token_perplexity": 148.20701048471787, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.0514983434490723, "token_perplexity": 30.25577056051104, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.2620381666076357, "token_perplexity": 199.10921984789258, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.3440385562528183, "token_perplexity": 101.51671686223926, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.5561483865320684, "token_perplexity": 245.8753494530723, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.0670613450542175, "token_perplexity": 47.01258283046883, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.2439340030318964, "token_perplexity": 100.77956982819596, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "f5cac1e7808e91851486ff0cae2677fd7ee3436d317a3bfd4d3d650ae132c958" }, { "step": 1500, "presented_tokens": 2304000, "components": { "bnc_spoken": { "bits_per_byte": 2.050397280026746, "token_perplexity": 122.43389947136116, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9962569087162336, "token_perplexity": 27.601581193544042, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.1821300101151504, "token_perplexity": 165.14836658404977, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.2709299059176744, "token_perplexity": 87.89320357610248, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.469325184667295, "token_perplexity": 203.9440164638083, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.9858696472279032, "token_perplexity": 40.41397243844651, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.166736851152362, "token_perplexity": 85.99066118101626, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "7550ae715d3a6bf02d845fa382004bc0a0a9b8ea4069dcc3632ac462904aefb8" }, { "step": 2000, "presented_tokens": 3072000, "components": { "bnc_spoken": { "bits_per_byte": 1.9971239595164685, "token_perplexity": 108.0572532404307, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9544191848849795, "token_perplexity": 25.74748086895464, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.133969763373271, "token_perplexity": 147.54522171498124, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.213701753112034, "token_perplexity": 78.5175979737359, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.408999778331554, "token_perplexity": 179.09761790887956, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.9012382491369528, "token_perplexity": 34.519644689004195, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.109375630441347, "token_perplexity": 76.42571165821566, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "52ec91ac309ec09c0eaf1c1b8de305f2f9bc478d25f101a1a9a7cb6b1ecc6588" }, { "step": 2500, "presented_tokens": 3840000, "components": { "bnc_spoken": { "bits_per_byte": 1.9820832455886783, "token_perplexity": 104.31291837545402, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.93736761491487, "token_perplexity": 25.02802705451744, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0975574474507055, "token_perplexity": 135.49285166001366, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1943172015114145, "token_perplexity": 75.57418867542968, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.3485190525529003, "token_perplexity": 157.2256445721014, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8713132835109578, "token_perplexity": 32.64807483802215, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0791332559670606, "token_perplexity": 71.81898420778984, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "4f0365b07137028e23ed493142e4099e5290b3f5c8c7a66bd77e66cbe51a29fa" }, { "step": 3000, "presented_tokens": 4608000, "components": { "bnc_spoken": { "bits_per_byte": 1.9422893837413444, "token_perplexity": 95.02029035214358, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8517810410519566, "token_perplexity": 21.70944370968915, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.080420025484918, "token_perplexity": 130.16620483812346, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1470658616992986, "token_perplexity": 68.85342193077636, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2926630004214954, "token_perplexity": 139.40627535388396, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.866393590313644, "token_perplexity": 32.35024969830179, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0387382364115605, "token_perplexity": 66.09594204454237, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "5d4c2ed23b86e4c31c9b9d0e350b50e7daf156fc831a0e985b4a49559de81a60" }, { "step": 3500, "presented_tokens": 5376000, "components": { "bnc_spoken": { "bits_per_byte": 1.925819257314017, "token_perplexity": 91.42078938488334, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8826232144032222, "token_perplexity": 22.85131135692818, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0472314313133886, "token_perplexity": 120.43868479812039, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1447287174816703, "token_perplexity": 68.53696884169977, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.269948686122649, "token_perplexity": 132.75108377879778, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8357869474874389, "token_perplexity": 30.55747276930818, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.024504602850375, "token_perplexity": 64.18995681294197, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "7f4cd145a7f94aa83a81a37499c3cd3f49c017560b0bb32000cf475b21ae66ae" }, { "step": 4000, "presented_tokens": 6144000, "components": { "bnc_spoken": { "bits_per_byte": 1.9116053538002244, "token_perplexity": 88.42418647046543, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9238532747700625, "token_perplexity": 24.472127304664504, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.028141708108961, "token_perplexity": 115.17642199533736, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.114274030198447, "token_perplexity": 64.54389259793071, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.227839123432153, "token_perplexity": 121.24220742868195, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8221293458172705, "token_perplexity": 29.789875498207692, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0094547747644476, "token_perplexity": 62.23442422950205, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "31dd323de37c190dace0d628faf19ec197567194ad296463d8ed7a11bc538026" }, { "step": 4500, "presented_tokens": 6912000, "components": { "bnc_spoken": { "bits_per_byte": 1.8837986513638239, "token_perplexity": 82.84298711482778, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8351377419076045, "token_perplexity": 21.117148010506593, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.012995844319906, "token_perplexity": 111.16539692274694, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.095322839617682, "token_perplexity": 62.177407030223726, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2187446077693336, "token_perplexity": 118.89070829423665, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7693498535012522, "token_perplexity": 27.00045373023055, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9767537470824719, "token_perplexity": 58.18825991732565, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "a826e3353276ccfc96373614be959c3e6d253b21bcf3e36a5cdb0c69e5eb8cd5" }, { "step": 5000, "presented_tokens": 7680000, "components": { "bnc_spoken": { "bits_per_byte": 1.8843285081208292, "token_perplexity": 82.94597158644378, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.874553547685801, "token_perplexity": 22.546871926688905, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.003733562792, "token_perplexity": 108.78164479524025, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0949318042111145, "token_perplexity": 62.12950217126229, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1968413125139437, "token_perplexity": 113.41283217089779, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7420132543473381, "token_perplexity": 25.659977013913803, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9723931572536573, "token_perplexity": 57.66897845046068, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "542c6286a0df920fcd4299c7f3ff6a240eae2e8c0feb3fa474e52470559342b2" }, { "step": 5500, "presented_tokens": 8448000, "components": { "bnc_spoken": { "bits_per_byte": 1.8633120377762524, "token_perplexity": 78.95768562969141, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8519208453049436, "token_perplexity": 21.714488729430823, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.991128244654097, "token_perplexity": 105.61943068445576, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.074620929409115, "token_perplexity": 59.69135169029513, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.178834162125899, "token_perplexity": 109.09892330045322, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7521626605510086, "token_perplexity": 26.14971400958837, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9580996726398532, "token_perplexity": 55.999108138629154, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "b77cff8b3a73494af10c8cec2bc383c090de83f1972529c110c422fefde5850b" }, { "step": 6000, "presented_tokens": 9216000, "components": { "bnc_spoken": { "bits_per_byte": 1.8634481854266278, "token_perplexity": 78.98289497675341, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.861560659428713, "token_perplexity": 22.06519646279363, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.977367839423903, "token_perplexity": 102.27229821863881, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.056893034952879, "token_perplexity": 57.641600226767146, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.171205286559901, "token_perplexity": 107.32115260141782, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7293314085703755, "token_perplexity": 25.060912840907243, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9492007335453128, "token_perplexity": 54.983981429334314, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "641714980ea5c05a448a17c7e157ec9209f5b4b954153c9689ce4d8608b26bb5" }, { "step": 6500, "presented_tokens": 9984000, "components": { "bnc_spoken": { "bits_per_byte": 1.846741442052141, "token_perplexity": 75.94876510219191, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8585386313843637, "token_perplexity": 21.95464629590772, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9711746539188117, "token_perplexity": 100.80066161697941, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0531818302262015, "token_perplexity": 57.22149148901705, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.153700015224645, "token_perplexity": 103.350598141697, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7272347998297586, "token_perplexity": 24.963229455482594, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9404765140154312, "token_perplexity": 54.006650960308335, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "3d98e3b059e73cdf06c145c9b60b35011139688771971435adff1b10b6c48cdf" }, { "step": 7000, "presented_tokens": 10752000, "components": { "bnc_spoken": { "bits_per_byte": 1.8397363327224656, "token_perplexity": 74.71150328297888, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9006534376787994, "token_perplexity": 23.546462251112143, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9633024296560413, "token_perplexity": 98.96057449156918, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.049466577118731, "token_perplexity": 56.80399135252807, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.136054037160662, "token_perplexity": 99.49678790025384, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.738968566178824, "token_perplexity": 25.514858666568752, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9414235304165606, "token_perplexity": 54.11189436240982, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "933fb99609b3821ea6577fce980a2b5850ad94744cecd1ec79a9ebeb9633687e" }, { "step": 7500, "presented_tokens": 11520000, "components": { "bnc_spoken": { "bits_per_byte": 1.8291509317505312, "token_perplexity": 72.88001501651411, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.844190856256609, "token_perplexity": 21.43729398897494, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9508191610871648, "token_perplexity": 96.11130135814389, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.036908211674722, "token_perplexity": 55.41517059153746, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.127815217649985, "token_perplexity": 97.74700416457493, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7069656417780494, "token_perplexity": 24.038280438460962, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9211560826618503, "token_perplexity": 51.903670014152674, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "f97175e16ab75de6c1c68ae68e362fddb5b6fa2856e8f43ad305086beb40ca9d" }, { "step": 8000, "presented_tokens": 12288000, "components": { "bnc_spoken": { "bits_per_byte": 1.8262814934898783, "token_perplexity": 72.39132562533565, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8542151208691586, "token_perplexity": 21.797448360041987, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9453852492078059, "token_perplexity": 94.89679300471698, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.027430120638851, "token_perplexity": 54.38952482206764, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.113090145141783, "token_perplexity": 94.6959459638102, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6947731478659591, "token_perplexity": 23.498486832539236, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9150034682069585, "token_perplexity": 51.251318808135125, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "5c6403e2c9a868c0ed394f8be9230c690005bef04dfb7acdaf11f5c972705d55" }, { "step": 8500, "presented_tokens": 13056000, "components": { "bnc_spoken": { "bits_per_byte": 1.817398690847142, "token_perplexity": 70.89918598628415, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7947562201078773, "token_perplexity": 19.746364926067436, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9421026531922208, "token_perplexity": 94.17056356481443, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.014635929453666, "token_perplexity": 53.035078443858424, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1106061143321884, "token_perplexity": 94.19072113861384, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6807409969649794, "token_perplexity": 22.89223252195651, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9000826800554131, "token_perplexity": 49.70314202312098, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "dc1474a4f4fe59f221bb0bb7db1c1fb272290568d4e35d71b1a68839fa0e99b4" }, { "step": 9000, "presented_tokens": 13824000, "components": { "bnc_spoken": { "bits_per_byte": 1.8165576788405373, "token_perplexity": 70.75951601834059, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8563896413842693, "token_perplexity": 21.876370261675156, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9327345281972557, "token_perplexity": 92.12841305856224, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0139463826636486, "token_perplexity": 52.96304567808903, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0982061196889052, "token_perplexity": 91.70872431818974, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6726721683908696, "token_perplexity": 22.550730588832437, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9029565893264684, "token_perplexity": 49.997657248764305, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "2240d11e028cd50f410d2a773d0d8ab5669f3d9993f33de38f69adf6505dfbf4" }, { "step": 9500, "presented_tokens": 14592000, "components": { "bnc_spoken": { "bits_per_byte": 1.811887506123757, "token_perplexity": 69.9889144588921, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8222054058324606, "token_perplexity": 20.668096104130314, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9287930532962478, "token_perplexity": 91.28250557297676, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.00814326126736, "token_perplexity": 52.36069220212956, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0918701566620825, "token_perplexity": 90.46586739072926, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.677793163925736, "token_perplexity": 22.766874130036825, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8954544711541412, "token_perplexity": 49.232490214404635, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "b8d1bcaeb84a7df550aed379fac8ac9dd9b955b85c219845ba7cb2fd3f475836" }, { "step": 10000, "presented_tokens": 15360000, "components": { "bnc_spoken": { "bits_per_byte": 1.808122833688161, "token_perplexity": 69.37383841706978, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8464154706433993, "token_perplexity": 21.51670325283877, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9225970248387834, "token_perplexity": 89.96840685499733, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0054381141326747, "token_perplexity": 52.08224863352299, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0822250941267346, "token_perplexity": 88.60615998488461, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6706268688051005, "token_perplexity": 22.464978499139857, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8936269355196393, "token_perplexity": 49.047874178059715, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "4dbf54e9fa78be5ed2a1536139eb45a3ca19c42c6334cb6a181af139c9481679" }, { "step": 10500, "presented_tokens": 16128000, "components": { "bnc_spoken": { "bits_per_byte": 1.8044539800977555, "token_perplexity": 68.77961853396371, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.896525560996783, "token_perplexity": 23.385469330800372, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.920246163485029, "token_perplexity": 89.47478440704955, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.001884206576797, "token_perplexity": 51.718690495753414, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.07764343608927, "token_perplexity": 87.73619211434283, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6704220031651376, "token_perplexity": 22.456407203488336, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8978369281441416, "token_perplexity": 49.47420774351442, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "45b41df15d47119c6b77ab99118892ee8012edb75da782c1248c4a62e927e07c" }, { "step": 11000, "presented_tokens": 16896000, "components": { "bnc_spoken": { "bits_per_byte": 1.8006990891808576, "token_perplexity": 68.17673361571566, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8705455818292633, "token_perplexity": 22.397176705134644, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9212456458929341, "token_perplexity": 89.68431904835532, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 1.9992144638740441, "token_perplexity": 51.447251032071335, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0728125886779507, "token_perplexity": 86.8281575214508, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6639068334388452, "token_perplexity": 22.18552058259384, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8915580073747136, "token_perplexity": 48.83970823562926, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "4b9a1daa8fd29a3f1efdb4e3354e7df70e1c7bd770639b5fabd390e391e4920e" }, { "step": 11500, "presented_tokens": 17664000, "components": { "bnc_spoken": { "bits_per_byte": 1.7997558864000862, "token_perplexity": 68.0261253390059, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8865087971847692, "token_perplexity": 22.999363167903184, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9153146375830654, "token_perplexity": 88.44806864825027, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 1.9981388193034262, "token_perplexity": 51.338290581122955, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.068618265734089, "token_perplexity": 86.04739325896605, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6626231951180772, "token_perplexity": 22.132536259721476, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8913063335119533, "token_perplexity": 48.814446327517366, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "df62c64a70d7eaa08454a97410b0d7417666dd59daacff8e49b84241fb546184" }, { "step": 12000, "presented_tokens": 18432000, "components": { "bnc_spoken": { "bits_per_byte": 1.7962104501070035, "token_perplexity": 67.4629686539784, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8796634677706432, "token_perplexity": 22.739176344430454, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.911031174294757, "token_perplexity": 87.56584153799601, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 1.9961187424928821, "token_perplexity": 51.13428446128243, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.06787510953647, "token_perplexity": 85.90979030063693, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.658368485546651, "token_perplexity": 21.95781914138275, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8878021206225761, "token_perplexity": 48.46406348206182, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "ee3577c4fcefe6e4e27b288baa3ef4864f406d6cc44d0bc76d50ded09799e298" } ], "source_commit": "426c09f74fb3b8ba40dfe322a2ae14b65cfdeb34", "protocol": { "steps": 12000, "batch": 16, "sequence": 96, "warmup": 16, "learning_rate": 0.002, "final_learning_rate": 0.0002, "lr_warmup_updates": 100, "weight_decay": 0.01, "eval_tokens": 49152, "threads": 4, "tokenizer_sha256": "46a2ef1bf8dce00e801c5ac188b13ec5c30c7f1ab7e8184031dca3ee057a7242", "train_cache_sha256": "4d205d6d35c87c38749ca3dc858fe96b28e3d7f4b8d7e2eb782e06909c8f15cd", "validation_cache_sha256": "ea3bd0f2625e78723baf6300dca2ffdae91deebdcb21ae748ffd0cc4ba2ef859", "graph_sha256": "a2f35369a5ed4981b8ef75c92df285f7df576a020582f119a1372bfbedbaebee", "validation_panel_sha256": "9a0c030150fee1821246dd54f045fc36c437bab3061e3010740d2942900de3ae", "evaluation_unit": "block", "cache_identity": "SHA-256 of verified mmap manifest" } }, { "scale": "100m", "seed": 42, "variant": "flm", "complete": true, "parameters": 600003, "validation": [ { "step": 0, "presented_tokens": 0, "components": { "bnc_spoken": { "bits_per_byte": 3.6282229809216866, "token_perplexity": 4949.802183341031, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 5.146004276643903, "token_perplexity": 5181.768185882155, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 3.689196108253878, "token_perplexity": 5618.583019089439, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 4.333268936301915, "token_perplexity": 5120.981294907628, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 3.980398153351656, "token_perplexity": 5281.868758404689, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 4.5422665140606595, "token_perplexity": 4727.254366275884, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 4.156506363683319, "token_perplexity": 5139.336632547325, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "a5c62e7c5c53916c99208b2599888e7712a716aa58745db8aab97605bfe46be6" }, { "step": 500, "presented_tokens": 768000, "components": { "bnc_spoken": { "bits_per_byte": 2.2328103095263483, "token_perplexity": 187.78084114282126, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.0163009013746858, "token_perplexity": 28.536589932723057, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.3892743631861184, "token_perplexity": 268.1709642043953, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.471523117310115, "token_perplexity": 130.51684110629978, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.7339963118808357, "token_perplexity": 360.62234612817866, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.0858977745523846, "token_perplexity": 48.691419093260215, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.336788813986303, "token_perplexity": 121.97555085757146, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "98e9b6c816e1467059a03402c87f77ff7d15366858b682a8a7e1d3f5343b9e4a" }, { "step": 1000, "presented_tokens": 1536000, "components": { "bnc_spoken": { "bits_per_byte": 2.104187151791172, "token_perplexity": 138.89142684431815, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9186666981399565, "token_perplexity": 24.262076031189455, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.2616024356065476, "token_perplexity": 198.9062832352731, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.3503180210573777, "token_perplexity": 102.78101315384357, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.5876325071945105, "token_perplexity": 263.12460731372545, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.9770688852403489, "token_perplexity": 39.7568427918172, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.2135991374618587, "token_perplexity": 94.68685493223397, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "47f0ac9d0abb1dcc9e6a0e93fbc0815228fc236b9af75676461e0ac024d87d8c" }, { "step": 1500, "presented_tokens": 2304000, "components": { "bnc_spoken": { "bits_per_byte": 2.047024484003584, "token_perplexity": 121.46948579164675, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8564814295310321, "token_perplexity": 21.879707892735087, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.2074551253656445, "token_perplexity": 175.23232960085298, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.288433613255013, "token_perplexity": 90.97849965202361, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.4954846554282253, "token_perplexity": 215.76321425280983, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.9097077744919622, "token_perplexity": 35.06856581391435, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.147928757572732, "token_perplexity": 82.72933264677768, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "6cf51e08ed823bb9d6e37040ecd250f62883c72a1b73347f8a5b6868cd0b389e" }, { "step": 2000, "presented_tokens": 3072000, "components": { "bnc_spoken": { "bits_per_byte": 2.0038347136905457, "token_perplexity": 109.7709508901895, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8034509893988024, "token_perplexity": 20.033793774363257, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.1679445214249804, "token_perplexity": 159.75572454481838, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.2507682286898882, "token_perplexity": 84.46884378551958, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.440722452526102, "token_perplexity": 191.76060835276093, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8684998505190757, "token_perplexity": 32.47742304725561, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.1032726257049266, "token_perplexity": 75.47285187209343, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "8dfa8a7369aed2b2adaad790f786add37f36975103cdab588124c447e5c4ebff" }, { "step": 2500, "presented_tokens": 3840000, "components": { "bnc_spoken": { "bits_per_byte": 1.9770816126098785, "token_perplexity": 103.09675040735381, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.775721299734266, "token_perplexity": 19.131429099789486, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.1349369481867435, "token_perplexity": 147.87956931978815, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.2175953844544325, "token_perplexity": 79.12250362264588, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.3927476032408577, "token_perplexity": 172.93761762525833, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.841779881172325, "token_perplexity": 30.900506995246914, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.070475072338678, "token_perplexity": 70.55199151537893, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "f70951574217535d919c7cac779b855f69f25d5eed5efbccbb9df5f3ff3eb2b8" }, { "step": 3000, "presented_tokens": 4608000, "components": { "bnc_spoken": { "bits_per_byte": 1.961889809052894, "token_perplexity": 99.48905264323538, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7515483097004558, "token_perplexity": 18.378028294386198, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.119845180082716, "token_perplexity": 142.74772614396596, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.185528661990562, "token_perplexity": 74.27631538543093, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.359287006384231, "token_perplexity": 160.9142250090941, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8240015230028772, "token_perplexity": 29.8939459266019, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0478151012072616, "token_perplexity": 67.34084692516778, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "dc8a1686aa6142251277a22b05dd26b036d2cabf668107c65d5dead5d3b090a5" }, { "step": 3500, "presented_tokens": 5376000, "components": { "bnc_spoken": { "bits_per_byte": 1.9459824889522177, "token_perplexity": 95.84666585490554, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7356366865836272, "token_perplexity": 17.89837576175634, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.1010785076371525, "token_perplexity": 136.61397248770555, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1797600746810355, "token_perplexity": 73.43656272546686, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.3316895649754, "token_perplexity": 151.62928854977105, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8083369776200624, "token_perplexity": 29.034270542602066, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.030980448900068, "token_perplexity": 65.05020729374303, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "537242a7892cde8da8a863760c9af60ca6673af0ab05e9844cc4c6587107e235" }, { "step": 4000, "presented_tokens": 6144000, "components": { "bnc_spoken": { "bits_per_byte": 1.942715434697178, "token_perplexity": 95.11525950257196, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7651105034509529, "token_perplexity": 18.79699171403265, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.085614427381707, "token_perplexity": 131.75822367206996, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1643316605150527, "token_perplexity": 71.23695568256895, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.3198496373079327, "token_perplexity": 147.81191624622352, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7913761951150975, "token_perplexity": 28.1313103696909, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0243027159187084, "token_perplexity": 64.16332182592107, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "e5c51384b7c063a089670425c5a7f4026f3a28c177f6c8b421905f2c713ceda4" }, { "step": 4500, "presented_tokens": 6912000, "components": { "bnc_spoken": { "bits_per_byte": 1.9283225960418533, "token_perplexity": 91.95896930302082, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7427559011228309, "token_perplexity": 18.111415665890778, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.072189353636895, "token_perplexity": 127.68290312336887, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.152034077322107, "token_perplexity": 69.53098912709922, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.3022529679646193, "token_perplexity": 142.3153124867152, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8041971831764594, "token_perplexity": 28.81123685162157, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.012905946262378, "token_perplexity": 62.677528605357935, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "196a2067b15c9f9fa92c8cebf48131122eab62efcbfcdee3f0d1e98535176c3f" }, { "step": 5000, "presented_tokens": 7680000, "components": { "bnc_spoken": { "bits_per_byte": 1.9205374143694134, "token_perplexity": 90.29558422755223, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7332959212975196, "token_perplexity": 17.828878083764213, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.062578304251275, "token_perplexity": 124.84302002604555, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1408582310838042, "token_perplexity": 68.01609431073277, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.288925603514318, "token_perplexity": 138.288737112434, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7521602796830305, "token_perplexity": 26.149598036959567, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9963562538150545, "token_perplexity": 60.58099265343359, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "39423dc5ebdc6460113fef79fba283dc2ce991e3ee929ae0c3b05f20a7dd81ee" }, { "step": 5500, "presented_tokens": 8448000, "components": { "bnc_spoken": { "bits_per_byte": 1.910294724828939, "token_perplexity": 88.15287295901476, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.735160050101948, "token_perplexity": 17.884202421128247, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.054333083363671, "token_perplexity": 122.45710402560904, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1283840453112783, "token_perplexity": 66.36415529908508, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2753694971110594, "token_perplexity": 134.30990829228733, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.767631521784736, "token_perplexity": 26.914168311296727, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9910721554774888, "token_perplexity": 59.92648018955245, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "1fb5c4e4f0d561e4a7035c45287306579871f0ec17f3a95d22f76e5c45e4d49a" }, { "step": 6000, "presented_tokens": 9216000, "components": { "bnc_spoken": { "bits_per_byte": 1.9053617920643058, "token_perplexity": 87.13914780669012, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7156173200406653, "token_perplexity": 17.31263944513323, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0474647316894585, "token_perplexity": 120.50446148600828, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1251379715656764, "token_perplexity": 65.9409008047677, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2549557888135525, "token_perplexity": 128.53325778863933, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7667387362907259, "token_perplexity": 26.869446310699953, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9819801455338488, "token_perplexity": 58.81681065893212, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "51c9a883fcad75c6178d516a5b6db6639923829deec42d0e4c360846b4fa0e91" }, { "step": 6500, "presented_tokens": 9984000, "components": { "bnc_spoken": { "bits_per_byte": 1.9047053681909543, "token_perplexity": 87.00513348298463, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7199943223697616, "token_perplexity": 17.439044386986335, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.04177067852534, "token_perplexity": 118.90929437455621, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1194523221973895, "token_perplexity": 65.20604530850473, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2432377689791134, "token_perplexity": 125.33024344312946, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7526766820479212, "token_perplexity": 26.174764161634016, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9762866117407696, "token_perplexity": 58.13240815239034, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "a98a2811f5e5470fc55b9cc2a915b86cd743b7f3d4e9cb5abb53e96c199c18b8" }, { "step": 7000, "presented_tokens": 10752000, "components": { "bnc_spoken": { "bits_per_byte": 1.8927999833249542, "token_perplexity": 84.61000641732596, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7132507171695655, "token_perplexity": 17.244675566231283, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0311253022595555, "token_perplexity": 115.98345712055094, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1084529528536424, "token_perplexity": 63.80757078449384, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.237443825642271, "token_perplexity": 123.77613651720488, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7374618197296596, "token_perplexity": 25.443346781130295, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9660278693795077, "token_perplexity": 56.91927540368337, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "adea79c2fe779ab6c2877a1e5284443f34600a5e526c1895e57e45f77d2a8faf" }, { "step": 7500, "presented_tokens": 11520000, "components": { "bnc_spoken": { "bits_per_byte": 1.887196644350365, "token_perplexity": 83.5056568952184, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7124825222288103, "token_perplexity": 17.22267202477326, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0296110019268654, "token_perplexity": 115.57314841288661, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.109168114818676, "token_perplexity": 63.89757875794999, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2262672503303107, "token_perplexity": 120.83248108742006, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7304891283734714, "token_perplexity": 25.11501603762363, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9616817324594664, "token_perplexity": 56.41299464246147, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "ba79985db29144326ae2112ed6c71a45cd3e82293d3366932ab1c85705d08b3a" }, { "step": 8000, "presented_tokens": 12288000, "components": { "bnc_spoken": { "bits_per_byte": 1.8822484375452853, "token_perplexity": 82.54241684294595, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7156324034523507, "token_perplexity": 17.31307346703649, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0200517862804035, "token_perplexity": 113.0163154795555, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0990480832671086, "token_perplexity": 62.63563430320083, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2200593178594357, "token_perplexity": 119.2278017323851, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7277215018415681, "token_perplexity": 24.98587146656391, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9562327548034257, "token_perplexity": 55.78460161148256, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "e0acf6df7b96d3cdc1437406d580f6e34d93139841a36a02b0608298c3f916ea" }, { "step": 8500, "presented_tokens": 13056000, "components": { "bnc_spoken": { "bits_per_byte": 1.8769362737198503, "token_perplexity": 81.52069071619523, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7191897553266027, "token_perplexity": 17.415739987683462, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0157183150459788, "token_perplexity": 111.87593785414643, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0984636063226643, "token_perplexity": 62.56351741825716, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2089182109217473, "token_perplexity": 116.401205992332, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7243691917310469, "token_perplexity": 24.83033306802777, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9523351884096294, "token_perplexity": 55.339420762568054, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "bee2f62b06b4f84a752fd0c902bdf5f484ec885fee8d55dbdbc8c9221e5d5472" }, { "step": 9000, "presented_tokens": 13824000, "components": { "bnc_spoken": { "bits_per_byte": 1.874327522107895, "token_perplexity": 81.0235712844748, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.703755998092049, "token_perplexity": 16.974678988656112, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0127501390570246, "token_perplexity": 111.10149245608346, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0905886835307035, "token_perplexity": 61.59991136650887, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2077505257537506, "token_perplexity": 116.1088616229021, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7120585963678183, "token_perplexity": 24.267414202750157, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.945874977843951, "token_perplexity": 54.609344933312656, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "87d519071a5d2ac16a8602868154a166999ffe91bb71f8bd08e5703a05fd5b28" }, { "step": 9500, "presented_tokens": 14592000, "components": { "bnc_spoken": { "bits_per_byte": 1.8714510342901134, "token_perplexity": 80.47894586372983, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7016910578511935, "token_perplexity": 16.916521271693085, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0088707697718076, "token_perplexity": 110.09737841137758, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0861909134895917, "token_perplexity": 61.068256355080514, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.204313777981061, "token_perplexity": 115.25268317948503, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7112172690203944, "token_perplexity": 24.229412552957523, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9428839887916358, "token_perplexity": 54.27459953003755, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "2972c147179dd52d55501be7eeccdcf741476b49d449a447564ac26cdffc500c" }, { "step": 10000, "presented_tokens": 15360000, "components": { "bnc_spoken": { "bits_per_byte": 1.871464591619611, "token_perplexity": 80.48150416124464, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.6898498938137136, "token_perplexity": 16.586849154236834, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.005115380907368, "token_perplexity": 109.13400017805958, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.084384454915876, "token_perplexity": 60.85120184795001, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1981490790781812, "token_perplexity": 113.73269337294904, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7069253390128845, "token_perplexity": 24.0364758584322, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.938565259196725, "token_perplexity": 53.79487324560033, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "c2bba80e27bb9d5a7eba4fed20b2b9d539fc27b075a6959da405794babf7d0f7" }, { "step": 10500, "presented_tokens": 16128000, "components": { "bnc_spoken": { "bits_per_byte": 1.870143995238669, "token_perplexity": 80.23268621523117, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7099327196171477, "token_perplexity": 17.149838637975343, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.002149665839114, "token_perplexity": 108.37915964293583, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0829297206860296, "token_perplexity": 60.676969591849165, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1951116169094167, "token_perplexity": 112.99115442107617, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7065646450137846, "token_perplexity": 24.020331602702413, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.939640374878274, "token_perplexity": 53.91389967875625, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "2358d41f1c7ae9133b79cad09138accaa0b46f896db59e35da7d742d5ea366b9" }, { "step": 11000, "presented_tokens": 16896000, "components": { "bnc_spoken": { "bits_per_byte": 1.8669286981107494, "token_perplexity": 79.63009314219161, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.6971135354659383, "token_perplexity": 16.78830774428117, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0016994494703897, "token_perplexity": 108.26502689433472, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.079032766722981, "token_perplexity": 60.212689001891604, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.190633945696521, "token_perplexity": 111.90682354900487, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7067203453800188, "token_perplexity": 24.0272992445805, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9358353578708745, "token_perplexity": 53.493824304317926, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "ce68504c6eced65654c405099884d60bb319a54d5397b09b551c12da672bb62f" }, { "step": 11500, "presented_tokens": 17664000, "components": { "bnc_spoken": { "bits_per_byte": 1.8670056760334217, "token_perplexity": 79.64446689990618, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.6978892356900028, "token_perplexity": 16.809966034044766, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9987466868676798, "token_perplexity": 107.51945588837195, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0795508177144186, "token_perplexity": 60.27420392890947, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1885783212646848, "token_perplexity": 111.41251646605006, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.704610065397488, "token_perplexity": 23.93303522934161, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9347994476316646, "token_perplexity": 53.38002732232092, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "5603e2aa3f4ef7d5ed828052d2e32dc3e4fd06b2d0fa1b3da3876031a0f6a9db" }, { "step": 12000, "presented_tokens": 18432000, "components": { "bnc_spoken": { "bits_per_byte": 1.8674238123242026, "token_perplexity": 79.72258903379706, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.6998088007112702, "token_perplexity": 16.86368231910664, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9970709097054167, "token_perplexity": 107.09860913427289, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0787302296387424, "token_perplexity": 60.17679388353562, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.185547484333604, "token_perplexity": 110.68768436947062, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7029058268994868, "token_perplexity": 23.857178660857766, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.933901950794822, "token_perplexity": 53.281631092804915, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "2ab4b6b32e20792a5055b2786724badb80d1a24d45b3403bf04edef1027b8deb" } ], "source_commit": "9fd9ec27bd022cd4818f61ab5a8584491c9ce786", "protocol": { "steps": 12000, "batch": 16, "sequence": 96, "warmup": 16, "learning_rate": 0.002, "final_learning_rate": 0.0002, "lr_warmup_updates": 100, "weight_decay": 0.01, "eval_tokens": 49152, "threads": 4, "tokenizer_sha256": "46a2ef1bf8dce00e801c5ac188b13ec5c30c7f1ab7e8184031dca3ee057a7242", "train_cache_sha256": "9dfb110a8a940b37c47f6b9212bdc0383a75c53810d2f145c70a05d081c10861", "validation_cache_sha256": "ea3bd0f2625e78723baf6300dca2ffdae91deebdcb21ae748ffd0cc4ba2ef859", "graph_sha256": "a2f35369a5ed4981b8ef75c92df285f7df576a020582f119a1372bfbedbaebee", "validation_panel_sha256": "9a0c030150fee1821246dd54f045fc36c437bab3061e3010740d2942900de3ae", "evaluation_unit": "block", "cache_identity": "SHA-256 of verified mmap manifest" } }, { "scale": "100m", "seed": 42, "variant": "gru", "complete": true, "parameters": 595408, "validation": [ { "step": 0, "presented_tokens": 0, "components": { "bnc_spoken": { "bits_per_byte": 3.548910474352327, "token_perplexity": 4109.837103544142, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 5.004167969041716, "token_perplexity": 4093.5301659483216, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 3.5565584390605474, "token_perplexity": 4119.234046030016, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 4.222534801199459, "token_perplexity": 4116.8292430438205, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 3.864499819140937, "token_perplexity": 4115.192808120306, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 4.468084879518217, "token_perplexity": 4117.156841240587, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 4.048016914079572, "token_perplexity": 4111.954062705375, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "4e54a20d52bb71017ea891fb2244e8f088723ed1cc17ae009446dc8e8faddf33" }, { "step": 500, "presented_tokens": 768000, "components": { "bnc_spoken": { "bits_per_byte": 2.256616665539686, "token_perplexity": 198.5606038773661, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.127641917111494, "token_perplexity": 34.33760593375132, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.40168399196625, "token_perplexity": 276.07345957779637, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.486266618966987, "token_perplexity": 134.36533726121624, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.756146754449223, "token_perplexity": 378.24178550149105, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.109240395177034, "token_perplexity": 50.85529491956946, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.3684205626018575, "token_perplexity": 130.1707620325346, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "f7a4f2e4c244e2bc2d53b3c0832df9c05700b6acee2d98f4bc414e787573be81" }, { "step": 1000, "presented_tokens": 1536000, "components": { "bnc_spoken": { "bits_per_byte": 2.1365612935910203, "token_perplexity": 149.84482399323215, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9887493167787804, "token_perplexity": 27.259309317263664, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.258500502369561, "token_perplexity": 197.46755912473947, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.3405170196782783, "token_perplexity": 100.81451557204332, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.580073017587086, "token_perplexity": 258.8756629271017, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.914400968248932, "token_perplexity": 35.376487610990125, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.2162580549769046, "token_perplexity": 95.20583341007601, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "957657b8dfddf2e6e5f7ec2eb4f3df275c31a16ed7a0f83176588112f5bb75c9" }, { "step": 1500, "presented_tokens": 2304000, "components": { "bnc_spoken": { "bits_per_byte": 2.065405401073098, "token_perplexity": 126.81899893422018, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9041262041023952, "token_perplexity": 23.682763090024103, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.176186413861626, "token_perplexity": 162.86708655644196, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.2605583474468567, "token_perplexity": 86.1146539320735, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.483279002531844, "token_perplexity": 210.16563178574887, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8297689101945775, "token_perplexity": 30.216833628024563, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.1332521114832264, "token_perplexity": 80.27055919000617, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "c565bcd7c65039a5bce97dc3408250dcc62c0e428a9eb25364c61f857f97f151" }, { "step": 2000, "presented_tokens": 3072000, "components": { "bnc_spoken": { "bits_per_byte": 2.006344321074603, "token_perplexity": 110.41877828296032, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8267086094387623, "token_perplexity": 20.823367568891637, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.119832588955463, "token_perplexity": 142.74351985208986, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.2088720197063414, "token_perplexity": 77.77368373981669, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.4176236374553097, "token_perplexity": 182.4549012758225, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7739479855295055, "token_perplexity": 27.232710261633144, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0727476569742884, "token_perplexity": 70.88236982855571, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "cab17e7f1caa0548360f943f37275a5ad40f1f20df58b2dbf8bbb305ab631dfa" }, { "step": 2500, "presented_tokens": 3840000, "components": { "bnc_spoken": { "bits_per_byte": 1.9646614726412595, "token_perplexity": 100.13771018573021, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8415329056167817, "token_perplexity": 21.342800630613155, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0779903342829478, "token_perplexity": 129.42815250883405, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1730455365162857, "token_perplexity": 72.4710544309809, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.349193984370824, "token_perplexity": 157.45433942040432, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7402981314528891, "token_perplexity": 25.57812824570002, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.036127895843683, "token_perplexity": 65.74220987419173, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "5dc84aebe0e8fb3bd8d8347655a6e2cb127d8f4126a93b446492f2f8a00d22e0" }, { "step": 3000, "presented_tokens": 4608000, "components": { "bnc_spoken": { "bits_per_byte": 1.942808638358453, "token_perplexity": 95.13604777218971, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8175868929675405, "token_perplexity": 20.510051319974025, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0475076099694602, "token_perplexity": 120.51655449190008, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.13759212711428, "token_perplexity": 67.57963601082315, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.3041995631259735, "token_perplexity": 142.91316674402256, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7146357104638568, "token_perplexity": 24.38419036330475, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0055664108930134, "token_perplexity": 61.738939478594894, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "f4cbf0546fafd961c36835f56672a022ce320bcb751a6e975cf0a4731b733863" }, { "step": 3500, "presented_tokens": 5376000, "components": { "bnc_spoken": { "bits_per_byte": 1.9188440796063275, "token_perplexity": 89.93778847263752, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7881593652179995, "token_perplexity": 19.531042762317806, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.029050379996177, "token_perplexity": 115.42161299625131, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1232195172400306, "token_perplexity": 65.6920244050562, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.269542015886956, "token_perplexity": 132.6348726463421, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6969591096518462, "token_perplexity": 23.594365319309357, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9825261121777333, "token_perplexity": 58.8828615436676, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "5baecc8796b608a900d9f76245e132d3b5e606612b271f654d54c871218d748b" }, { "step": 4000, "presented_tokens": 6144000, "components": { "bnc_spoken": { "bits_per_byte": 1.9110679358887506, "token_perplexity": 88.31283464779746, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8075781286646, "token_perplexity": 20.17168798395133, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.003363296101836, "token_perplexity": 108.68742240777792, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1087613215340233, "token_perplexity": 63.846365511180245, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.247855403960009, "token_perplexity": 126.58279062862567, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6806403095482132, "token_perplexity": 22.887939364391816, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9702294172600356, "token_perplexity": 57.41303199579935, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "a9f8ed9a8750a7f5c93140c508947910e57764a8b59d7b9fb0f7d73c072c0466" }, { "step": 4500, "presented_tokens": 6912000, "components": { "bnc_spoken": { "bits_per_byte": 1.893140152657833, "token_perplexity": 84.67751792538637, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7697300451123599, "token_perplexity": 18.941868523379956, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.989743324123177, "token_perplexity": 105.27765823221989, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0843725961085537, "token_perplexity": 60.84977950825826, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.219733847436301, "token_perplexity": 119.14426176467143, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6738215826545173, "token_perplexity": 22.59906499822744, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9493001701375612, "token_perplexity": 54.99522214914723, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "51e60e2a94f73ef6cd0364ef635d445803dd187559646c189b78e2547d0270d9" }, { "step": 5000, "presented_tokens": 7680000, "components": { "bnc_spoken": { "bits_per_byte": 1.8726306236403603, "token_perplexity": 80.70184149637828, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.783663916809622, "token_perplexity": 19.38565751302685, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9807001839224259, "token_perplexity": 103.07300676698318, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0780273422811124, "token_perplexity": 60.0934809929842, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.208071322184139, "token_perplexity": 116.18910373511066, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6518433623722886, "token_perplexity": 21.692544676636594, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.93919378336474, "token_perplexity": 53.86442543239969, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "e15e87afcada0ffd24855d5cd69d08c42e7df94c580454d4df9e06f3f6214552" }, { "step": 5500, "presented_tokens": 8448000, "components": { "bnc_spoken": { "bits_per_byte": 1.8614327604180132, "token_perplexity": 78.61053613887269, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7543080013646135, "token_perplexity": 18.462517103707444, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9644637144163426, "token_perplexity": 99.22989114767825, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0670961187907086, "token_perplexity": 58.81255478969557, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1864814629449483, "token_perplexity": 110.91054353689591, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6473919092105223, "token_perplexity": 21.51341514568052, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9238401669364729, "token_perplexity": 52.190854021275236, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "7f36e6df601fa16b004f857fdfb676e898fbd16deb3ee89540d3127fb74f60af" }, { "step": 6000, "presented_tokens": 9216000, "components": { "bnc_spoken": { "bits_per_byte": 1.8561282681694533, "token_perplexity": 77.63887611967877, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7419243533874713, "token_perplexity": 18.08640168356207, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.951918523288519, "token_perplexity": 96.35889874404492, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0558053548599045, "token_perplexity": 57.51815622617883, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1673095680949284, "token_perplexity": 106.42452974916583, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6456988260939422, "token_perplexity": 21.445673274646815, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9133754997327574, "token_perplexity": 51.0800840855037, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "19d8aa2368a901776dbe49fc5c552e9e9cf64a3993f0a082f673eec4fa7b2bca" }, { "step": 6500, "presented_tokens": 9984000, "components": { "bnc_spoken": { "bits_per_byte": 1.8489035914300427, "token_perplexity": 76.33477233435458, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7669429693063987, "token_perplexity": 18.854327939113574, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9481657927278109, "token_perplexity": 95.51632970137695, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0533355997113105, "token_perplexity": 57.238837250980836, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1538898239597066, "token_perplexity": 103.39285287710362, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6344417153137736, "token_perplexity": 21.000657513514803, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9102235416675848, "token_perplexity": 50.75017590437012, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "32870dac02bf65ba46d7fa0416aed2e68cb794de59f240cdeea4a2b7ec2fd384" }, { "step": 7000, "presented_tokens": 10752000, "components": { "bnc_spoken": { "bits_per_byte": 1.8409980612499115, "token_perplexity": 74.93285539758574, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7698706562645528, "token_perplexity": 18.946295800099037, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9377199547889132, "token_perplexity": 93.20960777893707, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0378675692475707, "token_perplexity": 55.520056729111886, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1472272081729455, "token_perplexity": 101.91992751803186, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6229910702653045, "token_perplexity": 20.55746378580098, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9017689119927155, "token_perplexity": 49.87573432401993, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "12f375c271c18ab8e7602f502027f31ced4e0cc5bf9f951d089ede13e0738ece" }, { "step": 7500, "presented_tokens": 11520000, "components": { "bnc_spoken": { "bits_per_byte": 1.8341593656374422, "token_perplexity": 73.74091090217543, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7590546128405664, "token_perplexity": 18.60874564590742, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9307926428930255, "token_perplexity": 91.71067630836468, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.03785500263511, "token_perplexity": 55.5186815445857, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.134608891190069, "token_perplexity": 99.18761417716343, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6195168779132945, "token_perplexity": 20.424854761267852, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8949672817918044, "token_perplexity": 49.183206951275864, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "e925bcdb3cd166ba968a950053096941ba6712b0db944ac5b6b5ac511a7d5eb4" }, { "step": 8000, "presented_tokens": 12288000, "components": { "bnc_spoken": { "bits_per_byte": 1.8260921738756404, "token_perplexity": 72.35919838157768, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7595151909073528, "token_perplexity": 18.622996140042044, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9225958811314594, "token_perplexity": 89.96816604398533, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0293724636241746, "token_perplexity": 54.598152067083866, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.127031719290321, "token_perplexity": 97.58221348851927, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6110359861885635, "token_perplexity": 20.104722990231032, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8879813184491145, "token_perplexity": 48.48192012751639, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "21b24bfeb22140a2f81f45459559b6d1140a5771e44bfe8c169da3d45209d03f" }, { "step": 8500, "presented_tokens": 13056000, "components": { "bnc_spoken": { "bits_per_byte": 1.8232311904759886, "token_perplexity": 71.87542612622931, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.744563349740811, "token_perplexity": 18.165905336142846, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.91837313143103, "token_perplexity": 89.08343349240286, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0258623152468647, "token_perplexity": 54.221708271873204, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1166281760193293, "token_perplexity": 95.4202251638433, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6063791334164437, "token_perplexity": 19.931077896618895, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.881540863881821, "token_perplexity": 47.84425624391731, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "cc392d2a2bb8d48d0623948f878bd1741b1895903b45280dd03bc81bef71fd76" }, { "step": 9000, "presented_tokens": 13824000, "components": { "bnc_spoken": { "bits_per_byte": 1.8179137764225768, "token_perplexity": 70.98486424522432, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7437581752818607, "token_perplexity": 18.141611292427292, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9175847525630398, "token_perplexity": 88.91922208280467, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0181925463250745, "token_perplexity": 53.40817538127952, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.111603701279688, "token_perplexity": 94.39329476711914, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6083377048113914, "token_perplexity": 20.003925767930895, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.878464856866299, "token_perplexity": 47.54266963878689, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "7a9f7dc646679119e27a49b3a78cf1563a7bdceb5837465e36d48cb229f24173" }, { "step": 9500, "presented_tokens": 14592000, "components": { "bnc_spoken": { "bits_per_byte": 1.8126326627425038, "token_perplexity": 70.11130390946906, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7294421112929024, "token_perplexity": 17.71504525795069, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.911944284912484, "token_perplexity": 87.75316588476956, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0131887835753055, "token_perplexity": 52.88401669229282, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.10505276658831, "token_perplexity": 93.07095831862826, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6012235043020495, "token_perplexity": 19.740583183835025, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8714438913502345, "token_perplexity": 46.861402641357365, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "f784f89836cb0bb19fc7194b53aff4eba27cf1fd34d9e5aefd69d047bed73258" }, { "step": 10000, "presented_tokens": 15360000, "components": { "bnc_spoken": { "bits_per_byte": 1.8127194698482232, "token_perplexity": 70.12557559390804, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7179610641296752, "token_perplexity": 17.380210851979136, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9072450804304255, "token_perplexity": 86.79338249983375, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.011781267224705, "token_perplexity": 52.73750448053884, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1008832865559333, "token_perplexity": 92.23899312200273, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.5961396902054297, "token_perplexity": 19.55452485570148, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8673010557405738, "token_perplexity": 46.463998615034726, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "8389adb9135d38b73386ce8bec6c95ecbf21b1115ffc02b5f35fc4a4ee343beb" }, { "step": 10500, "presented_tokens": 16128000, "components": { "bnc_spoken": { "bits_per_byte": 1.809814367533283, "token_perplexity": 69.64953136278092, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.738999043419895, "token_perplexity": 17.998678970658666, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.90141845750006, "token_perplexity": 85.61789544370865, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.00875187162653, "token_perplexity": 52.4235419396378, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0990713705408766, "token_perplexity": 91.87977106112744, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.5956317873692303, "token_perplexity": 19.53603316216795, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8676009633756538, "token_perplexity": 46.49265393788892, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "92cf8983015660fe668baf8ff8bf1846e54432f8fa02714457d0b9113b424f16" }, { "step": 11000, "presented_tokens": 16896000, "components": { "bnc_spoken": { "bits_per_byte": 1.8070991131124416, "token_perplexity": 69.20751869376004, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.722552334060152, "token_perplexity": 17.513344945517844, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9014930014304343, "token_perplexity": 85.63283324612064, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0054501902900395, "token_perplexity": 52.08348834862309, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0943771811980234, "token_perplexity": 90.95561652911667, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.593862497478404, "token_perplexity": 19.471753420549117, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8632683726881318, "token_perplexity": 46.080398213168934, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "17897b1bb28eeb40e2848f928bd31e81983d4cecb102e3793b264b7e816c9c73" }, { "step": 11500, "presented_tokens": 17664000, "components": { "bnc_spoken": { "bits_per_byte": 1.8045922686717804, "token_perplexity": 68.80192363464292, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7237763149829646, "token_perplexity": 17.549008851412133, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.899600133973565, "token_perplexity": 85.25432884987832, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0044067692807306, "token_perplexity": 51.97648155365586, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.091649957460117, "token_perplexity": 90.4229774985626, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.5922174256995396, "token_perplexity": 19.41217638710768, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8617062709819503, "token_perplexity": 45.93265902240819, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "40e1aaff87481eb1674ff4890a65ced2ea662acbd90606c744cfb0d419479142" }, { "step": 12000, "presented_tokens": 18432000, "components": { "bnc_spoken": { "bits_per_byte": 1.8049439958147415, "token_perplexity": 68.8586876715297, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7268819814920962, "token_perplexity": 17.63982688428143, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.8981400725877808, "token_perplexity": 84.96351320305766, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.002622649566389, "token_perplexity": 51.794022378631645, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0904012281639988, "token_perplexity": 90.18013722620695, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.5911378705621224, "token_perplexity": 19.373178896000248, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.861248423937446, "token_perplexity": 45.88944696406579, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "20225ca2104694ab9b22e82fb0900cd40cac82aa2238fd92c21725f42bf53613" } ], "source_commit": "57e6356cfe76635e48fc07bf7f9ca4f0cdb5f74d", "protocol": { "steps": 12000, "batch": 16, "sequence": 96, "warmup": 16, "learning_rate": 0.002, "final_learning_rate": 0.0002, "lr_warmup_updates": 100, "weight_decay": 0.01, "eval_tokens": 49152, "threads": 4, "tokenizer_sha256": "46a2ef1bf8dce00e801c5ac188b13ec5c30c7f1ab7e8184031dca3ee057a7242", "train_cache_sha256": "9dfb110a8a940b37c47f6b9212bdc0383a75c53810d2f145c70a05d081c10861", "validation_cache_sha256": "ea3bd0f2625e78723baf6300dca2ffdae91deebdcb21ae748ffd0cc4ba2ef859", "graph_sha256": "a2f35369a5ed4981b8ef75c92df285f7df576a020582f119a1372bfbedbaebee", "validation_panel_sha256": "9a0c030150fee1821246dd54f045fc36c437bab3061e3010740d2942900de3ae", "evaluation_unit": "block", "cache_identity": "SHA-256 of verified mmap manifest" } }, { "scale": "100m", "seed": 42, "variant": "transformer", "complete": true, "parameters": 607468, "validation": [ { "step": 0, "presented_tokens": 0, "components": { "bnc_spoken": { "bits_per_byte": 3.5717591360341667, "token_perplexity": 4336.0192509564895, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 5.039900792101366, "token_perplexity": 4344.007698997058, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 3.5868896909989223, "token_perplexity": 4422.262924176443, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 4.243492886783186, "token_perplexity": 4290.454865886343, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 3.889923762241634, "token_perplexity": 4346.790394255423, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 4.489396133784483, "token_perplexity": 4283.885151836662, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 4.073936771879149, "token_perplexity": 4336.999451555523, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "26150e1b52f79cd68b5589fd4e1f561625ec097a72e8c7db0463fc65d8b083d9" }, { "step": 500, "presented_tokens": 768000, "components": { "bnc_spoken": { "bits_per_byte": 2.25709257340395, "token_perplexity": 198.78229383027787, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.1628112880590584, "token_perplexity": 36.40456898922449, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.388146222061228, "token_perplexity": 267.4638753490941, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.4764124764474826, "token_perplexity": 131.7807363906525, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.7463494463736686, "token_perplexity": 370.344821776972, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.020536937886482, "token_perplexity": 43.1098719752933, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.3540001112978914, "token_perplexity": 126.36853305408427, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "6df85bbdbc7efa0389cc6af88984903e8319a9be34d9f6bb82fbd29a6bd32ed4" }, { "step": 1000, "presented_tokens": 1536000, "components": { "bnc_spoken": { "bits_per_byte": 2.1274975793251047, "token_perplexity": 146.69396411889338, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9385615366008722, "token_perplexity": 25.07774083138822, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.2494237274635176, "token_perplexity": 193.3171132721257, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.3451078645971957, "token_perplexity": 101.73090553787443, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.584970657946199, "token_perplexity": 261.6205715608538, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.91282009491019, "token_perplexity": 35.27246525109369, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.2074001429828742, "token_perplexity": 93.48786929141589, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "67fc24b6b3cd606aab6cd9ade1b619ebe3e435b198fb61d4d991e3c2862e592e" }, { "step": 1500, "presented_tokens": 2304000, "components": { "bnc_spoken": { "bits_per_byte": 2.063020942791022, "token_perplexity": 126.11195191769471, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8415085776057207, "token_perplexity": 21.341937666837204, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.1754081405273102, "token_perplexity": 162.57071124792344, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.2712240995952793, "token_perplexity": 87.94418509673865, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.4901878592022166, "token_perplexity": 213.31599173050682, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8415426823252756, "token_perplexity": 30.88685687601659, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.128902274315307, "token_perplexity": 79.55597023965466, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "ba5d73c2eaa8081662bee9e180154072b7f388f757f8d29f5fb05630259c6e8f" }, { "step": 2000, "presented_tokens": 3072000, "components": { "bnc_spoken": { "bits_per_byte": 2.002522674648953, "token_perplexity": 109.43377690999253, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7726279787829862, "token_perplexity": 19.03332214379599, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.116689999589377, "token_perplexity": 141.69754805279717, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.235907145384993, "token_perplexity": 82.03047098922667, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.421409277509681, "token_perplexity": 183.9484651626099, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.797685914233864, "token_perplexity": 28.463900164075476, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.072703736634212, "token_perplexity": 70.87597024616736, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "9f7610cbf262ba2bfc3c283f1a3ba31488d5ffec026932b38f017b7569d01bdc" }, { "step": 2500, "presented_tokens": 3840000, "components": { "bnc_spoken": { "bits_per_byte": 1.967297907706768, "token_perplexity": 100.75864423067372, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7487749242843564, "token_perplexity": 18.29350976685603, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.091322970688858, "token_perplexity": 133.5302871653466, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.184133462772373, "token_perplexity": 74.07233529144382, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.359629269968575, "token_perplexity": 161.032876344389, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7621727824781535, "token_perplexity": 26.64188477201683, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.033557106535703, "token_perplexity": 65.39568771456065, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "00cab8f9ca52da39f3610aa56d6ee055f2210ff60cf3e8bbdea2a91dce8794e2" }, { "step": 3000, "presented_tokens": 4608000, "components": { "bnc_spoken": { "bits_per_byte": 1.9488215205362118, "token_perplexity": 96.48681509193398, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.698660334347206, "token_perplexity": 16.83152353643571, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0566131836965225, "token_perplexity": 123.11229691319082, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1538271735932066, "token_perplexity": 69.77716638728509, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.317717298505135, "token_perplexity": 147.13470165992348, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7312036077547621, "token_perplexity": 25.148463748432057, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.999868525235128, "token_perplexity": 61.01998867480425, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "1dbad1264d9bb2c5910635c69bc1f294e165cd87fbc7cb23db857ddf260ed507" }, { "step": 3500, "presented_tokens": 5376000, "components": { "bnc_spoken": { "bits_per_byte": 1.915884813909078, "token_perplexity": 89.31590637617876, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.66154892735618, "token_perplexity": 15.82470967298023, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.037610807258941, "token_perplexity": 117.75728664319143, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.131788306417855, "token_perplexity": 66.81095465062063, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2771554864779704, "token_perplexity": 134.8274910752962, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7306806607527792, "token_perplexity": 25.123978087284453, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9743184480369103, "token_perplexity": 57.89767817183164, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "a3c10bd5dcf385e7769088ac6f381bc1069cf76c2142c3c1b57a0e5068c4ffc8" }, { "step": 4000, "presented_tokens": 6144000, "components": { "bnc_spoken": { "bits_per_byte": 1.910018885683884, "token_perplexity": 88.09587757949049, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.6977311618941602, "token_perplexity": 16.805550197359274, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0157004486980283, "token_perplexity": 111.87126012157873, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.115558304349916, "token_perplexity": 64.70748416932197, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.255903618150257, "token_perplexity": 128.79588904562175, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.702090700099212, "token_perplexity": 23.820981976303628, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.963299891040903, "token_perplexity": 56.60096531460793, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "99c6059bb202ef0a02d6bfb85188e2f54754ebcb135fc4d28e7f97ae064ea0bd" }, { "step": 4500, "presented_tokens": 6912000, "components": { "bnc_spoken": { "bits_per_byte": 1.8835815593225826, "token_perplexity": 82.80082942140595, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.637858025977157, "token_perplexity": 15.213713532279883, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.995515080937973, "token_perplexity": 106.70936040556965, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0960239910099716, "token_perplexity": 62.26339600543211, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.229580076751261, "token_perplexity": 121.69762815035814, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6879528320603703, "token_perplexity": 23.201837923588236, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9365238072384168, "token_perplexity": 53.569586111861724, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "8d272d1e9ffdde3bee47973b7259b2869e0889a9f8bf0b5d1deab62e84fdd9da" }, { "step": 5000, "presented_tokens": 7680000, "components": { "bnc_spoken": { "bits_per_byte": 1.8691044056125543, "token_perplexity": 80.03735493765559, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.6082245372052835, "token_perplexity": 14.482557181410309, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9854185759453067, "token_perplexity": 104.21749413874642, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0818244891046183, "token_perplexity": 60.54493052752307, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.208091898477153, "token_perplexity": 116.19425245887216, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6619843050452714, "token_perplexity": 22.106212183951193, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.917920357466152, "token_perplexity": 51.559563102126695, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "c595f8250b157d812a2f56562e2f183e67092d97f908d6679f17a411992d5f94" }, { "step": 5500, "presented_tokens": 8448000, "components": { "bnc_spoken": { "bits_per_byte": 1.8562018158478526, "token_perplexity": 77.65226589751292, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.6330381065876454, "token_perplexity": 15.0923243592744, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9725712662107526, "token_perplexity": 101.13066711274018, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0773024118491663, "token_perplexity": 60.00767617272617, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1875317291811665, "token_perplexity": 111.16168643923552, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6606712608550929, "token_perplexity": 22.05220930937463, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.911862937826475, "token_perplexity": 50.921500931745435, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "5a853dee6ea2e77fb0cdc9e22647c42e59e2b405150ca15c0f6559c87cfa4b3b" }, { "step": 6000, "presented_tokens": 9216000, "components": { "bnc_spoken": { "bits_per_byte": 1.8486209370775712, "token_perplexity": 76.28419896327024, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.6542266790456526, "token_perplexity": 15.633291329873837, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9518588571041464, "token_perplexity": 96.34544442369726, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.061509673651847, "token_perplexity": 58.168511643497645, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1679329804731844, "token_perplexity": 106.56750674193177, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6583835617106601, "token_perplexity": 21.958435793692285, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9030559818232027, "token_perplexity": 50.0078740499338, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "b6550565c01bc0ea1c9dda46f9d0f975a3949f93ed34a44c82f3a31aec8ee7a4" }, { "step": 6500, "presented_tokens": 9984000, "components": { "bnc_spoken": { "bits_per_byte": 1.8347416349063586, "token_perplexity": 73.84165429591587, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.6213516632315301, "token_perplexity": 14.802008401904455, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9503143699020222, "token_perplexity": 95.99782596208723, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.057568846165207, "token_perplexity": 57.71843342993261, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.144928786943197, "token_perplexity": 101.41669143659519, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6493768881411415, "token_perplexity": 21.593108594050552, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8896882680294362, "token_perplexity": 48.652343840385434, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "95dea00f55962ecc7b6aff3004c8f8ab37b11c087941678c67405790239c8912" }, { "step": 7000, "presented_tokens": 10752000, "components": { "bnc_spoken": { "bits_per_byte": 1.827596235100025, "token_perplexity": 72.61482906965533, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.59557370024578, "token_perplexity": 14.181221857588378, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9341880449029265, "token_perplexity": 92.442337031514, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0351991788400827, "token_perplexity": 55.22881345106781, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1321300645652186, "token_perplexity": 98.65953098477524, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6344085721516004, "token_perplexity": 20.999361028099962, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8737268021887556, "token_perplexity": 47.081842962327265, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "00f9cced962418d1b24a9e94c3f339d4215a0b5428523f01891ebcba75d5f490" }, { "step": 7500, "presented_tokens": 11520000, "components": { "bnc_spoken": { "bits_per_byte": 1.821803773523838, "token_perplexity": 71.63527068109522, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.5550699612336767, "token_perplexity": 13.257979028115873, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.930857336773407, "token_perplexity": 91.72456265385487, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.035197544934763, "token_perplexity": 55.228635586216946, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1227812660527334, "token_perplexity": 96.69305910588285, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6338164899832424, "token_perplexity": 20.97621359703763, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8648139414457787, "token_perplexity": 46.227041434202086, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "c52be82741b72e281936f5e8d1ba4b038a97c66ceadfe8ce011e664876228e2b" }, { "step": 8000, "presented_tokens": 12288000, "components": { "bnc_spoken": { "bits_per_byte": 1.8121367159917647, "token_perplexity": 70.02982253910012, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.585560619032087, "token_perplexity": 13.947167934319559, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9118244742082469, "token_perplexity": 87.72856393536705, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.023203526041257, "token_perplexity": 53.93829652474131, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.109545553030155, "token_perplexity": 93.97583645377327, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6201915405024725, "token_perplexity": 20.450539357104127, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.857188387773195, "token_perplexity": 45.508030958123584, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "5bdce9a6f8d69ce70301e073419d95a233e1c3756fa0a3696d6c65d8c5e31d99" }, { "step": 8500, "presented_tokens": 13056000, "components": { "bnc_spoken": { "bits_per_byte": 1.8048905966508186, "token_perplexity": 68.85006674930759, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.5883156620602634, "token_perplexity": 14.011178615576863, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9080724161824645, "token_perplexity": 86.96159602398025, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.014086312621048, "token_perplexity": 52.977655394242866, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0922979039757266, "token_perplexity": 90.54924117638792, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6126116658153034, "token_perplexity": 20.163818940990947, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8498572152992814, "token_perplexity": 44.827323838754126, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "6f8daf98685cfc9243de7b676e6a3405aeb770ca7042ac5b06fc01fe035802aa" }, { "step": 9000, "presented_tokens": 13824000, "components": { "bnc_spoken": { "bits_per_byte": 1.7977180424651993, "token_perplexity": 67.7018624664787, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.566946022541123, "token_perplexity": 13.522272429150453, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9072193354751, "token_perplexity": 86.78815327078597, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.011131239397425, "token_perplexity": 52.66997839593248, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0840934016394055, "token_perplexity": 88.96338650756856, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6117578921580413, "token_perplexity": 20.131776584396082, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8434187101463397, "token_perplexity": 44.23790472430261, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "2901e48b61c227af76e952ab2716463a71a702176c574acd413d4a1d5c943d7e" }, { "step": 9500, "presented_tokens": 14592000, "components": { "bnc_spoken": { "bits_per_byte": 1.7930008941498494, "token_perplexity": 66.95718474451189, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.5299519902465435, "token_perplexity": 12.71588895693691, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9028590034345017, "token_perplexity": 85.90702728915433, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0024993525572463, "token_perplexity": 51.781436670875834, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0760568257256935, "token_perplexity": 87.43692103582, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.601165066542491, "token_perplexity": 19.738434442661625, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8323275018646445, "token_perplexity": 43.24066619846698, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "709ba9ecea7dd856fde58a9bd6f4ed87505a340094f30837b33bf5bae51fc4c2" }, { "step": 10000, "presented_tokens": 15360000, "components": { "bnc_spoken": { "bits_per_byte": 1.7891423253311145, "token_perplexity": 66.35414205220766, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.5176074169237985, "token_perplexity": 12.45765194082101, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.8943189567656418, "token_perplexity": 84.20710845044158, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0009246193689036, "token_perplexity": 51.620962373291746, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0627463893631126, "token_perplexity": 84.96613603474181, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.597624260884266, "token_perplexity": 19.608675420041674, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.825199874510041, "token_perplexity": 42.61170071851632, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "363c46913b1cbfcf981284877d402abd792eeeda056ce1a3f313b05387cb6a91" }, { "step": 10500, "presented_tokens": 16128000, "components": { "bnc_spoken": { "bits_per_byte": 1.7879040929442143, "token_perplexity": 66.1617763576214, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.5462175507345388, "token_perplexity": 13.064340751193425, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.8857114735473415, "token_perplexity": 82.52780169511041, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 1.9956130483238637, "token_perplexity": 51.083341763792205, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0594365071078378, "token_perplexity": 84.36264762902184, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.5964004557456786, "token_perplexity": 19.56402558963634, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.825574997690659, "token_perplexity": 42.64457358320276, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "00549b393b3571ef3d8141a87f74a66f2b8905d9ef54573d00eb9c4af0a6e7e1" }, { "step": 11000, "presented_tokens": 16896000, "components": { "bnc_spoken": { "bits_per_byte": 1.7822320283344728, "token_perplexity": 65.28769605102974, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.5168516639205227, "token_perplexity": 12.442013743847243, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.884111761279054, "token_perplexity": 82.21941087665547, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 1.9943421619900161, "token_perplexity": 50.9555389530758, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0544839499286898, "token_perplexity": 83.46764733996916, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.5914066806562912, "token_perplexity": 19.382881971369123, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.81841427136735, "token_perplexity": 42.021419334075226, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "7640be77f6ec89c711a802b25dd4d9b82032328f026385c20290b0b4e68f2586" }, { "step": 11500, "presented_tokens": 17664000, "components": { "bnc_spoken": { "bits_per_byte": 1.7793304357043866, "token_perplexity": 64.84502716005368, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.5147363289018256, "token_perplexity": 12.39834708630338, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.8821152892199156, "token_perplexity": 81.83614919893857, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 1.9906959600460665, "token_perplexity": 50.590641628949584, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0505673671544957, "token_perplexity": 82.7665914376102, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.5899624907382948, "token_perplexity": 19.330808942819043, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8157142544925355, "token_perplexity": 41.78882464725983, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "b3f5a02bacc13554db7da0a0655ab612d0cbc7214c7cfded0c8abdf63a1acf4a" }, { "step": 12000, "presented_tokens": 18432000, "components": { "bnc_spoken": { "bits_per_byte": 1.7787498092666787, "token_perplexity": 64.75680745345342, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.534615271448383, "token_perplexity": 12.814827559618525, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.881515956338068, "token_perplexity": 81.72144462312664, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 1.9867592263002287, "token_perplexity": 50.199602158449785, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.048897423236668, "token_perplexity": 82.4694702038678, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.591638136821779, "token_perplexity": 19.39124059821724, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8175012214235573, "token_perplexity": 41.942619546073615, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "1cfe5b992296e7f0ece6f0a30fc518d6109d9e76afbec36498d600e994d6bed9" } ], "source_commit": "52dce9a89edc1530b38698dfe18df81fd8f2b6cc", "protocol": { "steps": 12000, "batch": 16, "sequence": 96, "warmup": 16, "learning_rate": 0.002, "final_learning_rate": 0.0002, "lr_warmup_updates": 100, "weight_decay": 0.01, "eval_tokens": 49152, "threads": 4, "tokenizer_sha256": "46a2ef1bf8dce00e801c5ac188b13ec5c30c7f1ab7e8184031dca3ee057a7242", "train_cache_sha256": "9dfb110a8a940b37c47f6b9212bdc0383a75c53810d2f145c70a05d081c10861", "validation_cache_sha256": "ea3bd0f2625e78723baf6300dca2ffdae91deebdcb21ae748ffd0cc4ba2ef859", "graph_sha256": "a2f35369a5ed4981b8ef75c92df285f7df576a020582f119a1372bfbedbaebee", "validation_panel_sha256": "9a0c030150fee1821246dd54f045fc36c437bab3061e3010740d2942900de3ae", "evaluation_unit": "block", "cache_identity": "SHA-256 of verified mmap manifest" } }, { "scale": "100m", "seed": 43, "variant": "flm", "complete": true, "parameters": 600003, "validation": [ { "step": 0, "presented_tokens": 0, "components": { "bnc_spoken": { "bits_per_byte": 3.653515902165921, "token_perplexity": 5252.225987162989, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 5.073145501114346, "token_perplexity": 4590.78772996143, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 3.663569060637094, "token_perplexity": 5291.514275372805, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 4.349295118839944, "token_perplexity": 5285.327207653864, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 3.9872834982124745, "token_perplexity": 5360.771951201961, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 4.508799105867208, "token_perplexity": 4441.549044437215, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 4.14534326597165, "token_perplexity": 5022.740146813829, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "a26dd318518f18224d2c7337a3c3b72182a7c9ee5860d012940142a46ed7fb0c" }, { "step": 500, "presented_tokens": 768000, "components": { "bnc_spoken": { "bits_per_byte": 2.2373621070394614, "token_perplexity": 189.7956855730298, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.0368041833025807, "token_perplexity": 29.52580249657868, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.393632163470978, "token_perplexity": 270.9199186425953, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.4674671226509246, "token_perplexity": 129.47757367760505, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.7563092682429255, "token_perplexity": 378.374187080861, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.170977911760231, "token_perplexity": 57.05328796481713, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.3573401939639895, "token_perplexity": 127.23920656942553, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "cb6a35545fd94239e04b7401fbb44fb21762226f4a687574453bd5acbbd710f4" }, { "step": 1000, "presented_tokens": 1536000, "components": { "bnc_spoken": { "bits_per_byte": 2.137145884113704, "token_perplexity": 150.05035573741486, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.905277855476936, "token_perplexity": 23.72813765167356, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.2694508306436574, "token_perplexity": 202.59347674761142, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.3589947736713417, "token_perplexity": 104.55392588980412, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.5749817559540458, "token_perplexity": 256.0527637396695, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.9680901631208905, "token_perplexity": 39.097435297051504, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.2173712013781777, "token_perplexity": 95.42394571298448, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "49f541c3e69b4443a64b443ce2d143cf6bd112c888b12dbd60e24b5e79d0f614" }, { "step": 1500, "presented_tokens": 2304000, "components": { "bnc_spoken": { "bits_per_byte": 2.0511337164529526, "token_perplexity": 122.64549167750393, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8874269633085068, "token_perplexity": 23.03448780345066, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.2037621397833416, "token_perplexity": 173.7243743598763, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.298584607737643, "token_perplexity": 92.81714324190833, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.501703132924927, "token_perplexity": 218.67212339808816, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.9236236948380603, "token_perplexity": 35.98949272872406, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.1569885336956114, "token_perplexity": 84.2845634205843, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "fba05a71f4cded06b8f3beeb51155dd93963f3c8658444f14a84631f10105fa6" }, { "step": 2000, "presented_tokens": 3072000, "components": { "bnc_spoken": { "bits_per_byte": 2.0178476301520165, "token_perplexity": 113.43750268908545, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8099425226491233, "token_perplexity": 20.251113314707577, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.169601825764011, "token_perplexity": 160.37655524746984, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.2598021101037804, "token_perplexity": 85.9863884564312, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.4448302901707897, "token_perplexity": 193.46454132577819, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8509801680057638, "token_perplexity": 31.43463816789131, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.106640594404304, "token_perplexity": 75.99721294632198, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "9d9daa29bcc90e43aab6bfa99b5125747021a95a16172cc05a3ee6cae018427d" }, { "step": 2500, "presented_tokens": 3840000, "components": { "bnc_spoken": { "bits_per_byte": 1.9845583837047813, "token_perplexity": 104.92005448733103, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.766866440520176, "token_perplexity": 18.85192992576456, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.1341661435716532, "token_perplexity": 147.6130474701386, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.2295321741367324, "token_perplexity": 81.00617379764772, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.4068457198200477, "token_perplexity": 178.26872618615516, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8399088209247825, "token_perplexity": 30.792996471502804, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.074643376278947, "token_perplexity": 71.15914434311642, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "36892be35f3062fd3d3ff5bdde164b267dfe3715dec07e4aedbc2a694d270481" }, { "step": 3000, "presented_tokens": 4608000, "components": { "bnc_spoken": { "bits_per_byte": 1.966397882470404, "token_perplexity": 100.54623834110828, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7569560854202608, "token_perplexity": 18.543954162257, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.114419075975693, "token_perplexity": 140.94647305851802, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.21261954173513, "token_perplexity": 78.35029093247172, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.361486401933451, "token_perplexity": 161.6782087980175, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.826793551181681, "token_perplexity": 30.049824738803927, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0535060297407237, "token_perplexity": 68.13329700819317, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "4fe3f5af90f4f25f169fcae14a8f2b799a6dac213e7afc1d3344d8ae9f99bee4" }, { "step": 3500, "presented_tokens": 5376000, "components": { "bnc_spoken": { "bits_per_byte": 1.9559347588234597, "token_perplexity": 98.10955812118918, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.727035618462999, "token_perplexity": 17.644331822495502, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0905581814808336, "token_perplexity": 133.29150332907395, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1880439085030634, "token_perplexity": 74.64546815524442, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.341816289209639, "token_perplexity": 154.97242252902873, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8180628123112637, "token_perplexity": 29.565071819511616, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.034277903285056, "token_perplexity": 65.49266076322604, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "89abe87b361577865c21f545995b05e64e7c26502ebb615bbf6a96eeb64a3e78" }, { "step": 4000, "presented_tokens": 6144000, "components": { "bnc_spoken": { "bits_per_byte": 1.9395495083013354, "token_perplexity": 94.41181829421099, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7469113580679856, "token_perplexity": 18.236936299753108, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.084397680377245, "token_perplexity": 131.3835684274964, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1712669412730254, "token_perplexity": 72.21743711206868, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.325741708545054, "token_perplexity": 149.69944315009982, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7952024649440035, "token_perplexity": 28.332529085274487, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0237325693788923, "token_perplexity": 64.08816193123076, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "479fb0612c8bd5637b0c27912a8046a91869e83c18224fcef41d46ec1e929ca4" }, { "step": 4500, "presented_tokens": 6912000, "components": { "bnc_spoken": { "bits_per_byte": 1.931084726760538, "token_perplexity": 92.55646271953496, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7494413871663281, "token_perplexity": 18.313784581479883, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0670218308519637, "token_perplexity": 126.1480610254317, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1621417558312555, "token_perplexity": 70.93012991083668, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.3020164452494947, "token_perplexity": 142.24284030832544, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7856364233920876, "token_perplexity": 27.83213958933921, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.012122267044829, "token_perplexity": 62.57663405125412, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "86f830091b53e342b62208e5f6bbaade6a0b41c1b57c211192f32173c85eefec" }, { "step": 5000, "presented_tokens": 7680000, "components": { "bnc_spoken": { "bits_per_byte": 1.9312826450232279, "token_perplexity": 92.59942438485737, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7387960331831314, "token_perplexity": 17.99260701437015, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0569108134310845, "token_perplexity": 123.19807986851227, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1568036283910157, "token_perplexity": 70.18773469626187, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2814919259596222, "token_perplexity": 136.09251472886027, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7615785624283433, "token_perplexity": 26.612411651501144, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.000737198555982, "token_perplexity": 61.129053260629604, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "c090794c7252b59c31cbf9a5320d8efb7a97f3307923a87299f52d8dfa7401f9" }, { "step": 5500, "presented_tokens": 8448000, "components": { "bnc_spoken": { "bits_per_byte": 1.9165524301658408, "token_perplexity": 89.45582753303947, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.736448338650092, "token_perplexity": 17.922537035156335, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0562101991770843, "token_perplexity": 122.99624374199618, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.150871832934532, "token_perplexity": 69.37188672310988, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2719986436409996, "token_perplexity": 133.33843701169405, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.759964690919388, "token_perplexity": 26.532528578714352, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9946386452012987, "token_perplexity": 60.36746142360612, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "de8bddded3a658e8fa48d4f6145ae0653e6bb2f77a133d6a1b069f1fe4305a9a" }, { "step": 6000, "presented_tokens": 9216000, "components": { "bnc_spoken": { "bits_per_byte": 1.9131679659101888, "token_perplexity": 88.7487545564009, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.727425795694755, "token_perplexity": 17.65577775814396, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0441719561215623, "token_perplexity": 119.57941199845904, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1383916208958262, "token_perplexity": 67.68621495309954, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.254784477119365, "token_perplexity": 128.48584670873697, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7706247398073298, "token_perplexity": 27.06465040119189, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9871053713586302, "token_perplexity": 59.43978719592644, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "47c2277b43b9ae414e999e030fc97cac3c4d2dfdecabce2b9ace1a635f34b37f" }, { "step": 6500, "presented_tokens": 9984000, "components": { "bnc_spoken": { "bits_per_byte": 1.900944230861198, "token_perplexity": 86.24123046634595, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7247051108830476, "token_perplexity": 17.576120221822876, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.037178469972557, "token_perplexity": 117.6382000741568, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.130867861117626, "token_perplexity": 66.68985301882239, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.249373861731714, "token_perplexity": 126.99740571623421, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7539090357124831, "token_perplexity": 26.23491904877145, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9784167734534885, "token_perplexity": 58.38753105730884, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "c67b53984dce64245aa4cbf9af792cad62850e527bdc2f9379b6785efad54e55" }, { "step": 7000, "presented_tokens": 10752000, "components": { "bnc_spoken": { "bits_per_byte": 1.8967558009193322, "token_perplexity": 85.39843315789386, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7173769128115692, "token_perplexity": 17.36334481968486, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0325875157571027, "token_perplexity": 116.38103491887603, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.123579628460543, "token_perplexity": 65.73866903705968, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.240072963860662, "token_perplexity": 124.47894637374357, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7489805430031122, "token_perplexity": 25.99517046529564, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9722291474624756, "token_perplexity": 57.649538008245266, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "2771a805525ddf9e0691b8a81edce34458c6388335ea2be8fce6133c3e927bb9" }, { "step": 7500, "presented_tokens": 11520000, "components": { "bnc_spoken": { "bits_per_byte": 1.885676836414079, "token_perplexity": 83.2086142889763, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7095449014448758, "token_perplexity": 17.138787892710365, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0213408200447662, "token_perplexity": 113.35776984418986, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1223380021674245, "token_perplexity": 65.577982784842, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.227768928779423, "token_perplexity": 121.22388080461691, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7367130342506818, "token_perplexity": 25.40788316507065, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9627345089307715, "token_perplexity": 56.535217570394536, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "d90276f1a2a8b1f3ab48c93a8f0c862b8957036ab70f1d174e183dda3fa6fc14" }, { "step": 8000, "presented_tokens": 12288000, "components": { "bnc_spoken": { "bits_per_byte": 1.8835244614052236, "token_perplexity": 82.78974498541614, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.6841003660214462, "token_perplexity": 16.429100376421896, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.017011105455135, "token_perplexity": 112.21493336834115, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.109329080480194, "token_perplexity": 63.917854874054456, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2197951040725523, "token_perplexity": 119.15998031007868, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.732176722794406, "token_perplexity": 25.194090917899047, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9539190977636944, "token_perplexity": 55.519905089682624, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "5772d78b3758fe94f1fad8370bd0570e51741c514598ad3537d5e1d69859d6d0" }, { "step": 8500, "presented_tokens": 13056000, "components": { "bnc_spoken": { "bits_per_byte": 1.8787229932687355, "token_perplexity": 81.86292378664471, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.694391276080086, "token_perplexity": 16.71252023364617, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0155186079884357, "token_perplexity": 111.82366206369569, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.110070180810306, "token_perplexity": 64.01129096618361, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.212398892842396, "token_perplexity": 117.27701210605436, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7244168290895001, "token_perplexity": 24.832536520527025, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9517650341681994, "token_perplexity": 55.27459617010795, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "870d41720a64767009d4de138762a0b0b6453cecff36188c8e32d725eaa3fe88" }, { "step": 9000, "presented_tokens": 13824000, "components": { "bnc_spoken": { "bits_per_byte": 1.8749867812155712, "token_perplexity": 81.14891164452207, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.6893144130867432, "token_perplexity": 16.572093521813855, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.008691176681897, "token_perplexity": 110.05111396943528, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1031421151287466, "token_perplexity": 63.14312068152565, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2066535126707554, "token_perplexity": 115.83487983224266, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7203078579367097, "token_perplexity": 24.64319432796217, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.946344498269724, "token_perplexity": 54.66207986454151, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "81de5a4254c8a869aafca790d3e2d56de12b1e8b7975bc1856a0c3b568b34f5d" }, { "step": 9500, "presented_tokens": 14592000, "components": { "bnc_spoken": { "bits_per_byte": 1.8687997597241353, "token_perplexity": 79.98020435469918, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.6880618387940112, "token_perplexity": 16.53762900319338, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.008191081302746, "token_perplexity": 109.92238830042481, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0990604684395335, "token_perplexity": 62.63716337215349, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2030748671833997, "token_perplexity": 114.94559063040896, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7130409968925715, "token_perplexity": 24.31186338017071, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.942530191303994, "token_perplexity": 54.23513919532834, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "d950edc9120d86a1a0379e1a98916f5a595e79b0ac8acf614c6924fd7ebd1514" }, { "step": 10000, "presented_tokens": 15360000, "components": { "bnc_spoken": { "bits_per_byte": 1.8683229255827007, "token_perplexity": 79.89083372597194, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.6738818930976387, "token_perplexity": 16.15243144240343, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.00261983222021, "token_perplexity": 108.49847828409195, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0954723944019658, "token_perplexity": 62.195738410302056, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.200886601341557, "token_perplexity": 114.40517623345168, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7126109587386853, "token_perplexity": 24.29239608975584, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9384513532353913, "token_perplexity": 53.78227808199776, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "69c566750712284857cf1b333cb63616f7c3934422b64af2de0a146b6dccc560" }, { "step": 10500, "presented_tokens": 16128000, "components": { "bnc_spoken": { "bits_per_byte": 1.867896213681236, "token_perplexity": 79.81094192990597, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.6799061050693362, "token_perplexity": 16.31497040792823, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9983009116171653, "token_perplexity": 107.40734482200459, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.092081776263061, "token_perplexity": 61.78146473635498, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.19680727470666, "token_perplexity": 113.40451902723929, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7166879176665903, "token_perplexity": 24.477583318542226, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9377241691300207, "token_perplexity": 53.70193910154475, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "538d5f04b606c42eb4072322188f8f3d623e30dc2e4e306481a8a422c4467352" }, { "step": 11000, "presented_tokens": 16896000, "components": { "bnc_spoken": { "bits_per_byte": 1.8646145169383663, "token_perplexity": 79.19918512641658, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.6797987404058174, "token_perplexity": 16.312059340643884, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9982792042590933, "token_perplexity": 107.40188847589162, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0893573133245935, "token_perplexity": 61.450583785374455, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1900846097029425, "token_perplexity": 111.77451278106803, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7151272932050246, "token_perplexity": 24.406529070216006, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9352369302502976, "token_perplexity": 53.42805617558929, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "95278b0aa07ed433d91cf141c007d12d69e0583cf2af603eab5efc70fd2aa990" }, { "step": 11500, "presented_tokens": 17664000, "components": { "bnc_spoken": { "bits_per_byte": 1.864894410730659, "token_perplexity": 79.25117804655399, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.6836069931054023, "token_perplexity": 16.415633887705262, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9961749212697755, "token_perplexity": 106.87427106707784, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0871865070945357, "token_perplexity": 61.18821237368686, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.188130292593542, "token_perplexity": 111.3050710916627, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7097120594860067, "token_perplexity": 24.16157262914258, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9338981046693187, "token_perplexity": 53.281209816955275, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "77f16f1bfe8a3119a69d8a77d1379e9d3ea93658677f37b7f2089fe31e706ecd" }, { "step": 12000, "presented_tokens": 18432000, "components": { "bnc_spoken": { "bits_per_byte": 1.8642678644596986, "token_perplexity": 79.13483843011655, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.6755479158193665, "token_perplexity": 16.197219639617952, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9956182724684952, "token_perplexity": 106.73513376346065, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0872891019846245, "token_perplexity": 61.20058710107617, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1884016817031178, "token_perplexity": 111.3701427265198, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.704490697016603, "token_perplexity": 23.92771423241688, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9318626803563625, "token_perplexity": 53.05873131008212, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "c23423b212996be5c80fd31a030548ef4bcfa1f5641b473ff106de4f1645853d" } ], "source_commit": "e8cf5eaa02d35aa5a1b100a11c59fe3225d58a9c", "protocol": { "steps": 12000, "batch": 16, "sequence": 96, "warmup": 16, "learning_rate": 0.002, "final_learning_rate": 0.0002, "lr_warmup_updates": 100, "weight_decay": 0.01, "eval_tokens": 49152, "threads": 4, "tokenizer_sha256": "46a2ef1bf8dce00e801c5ac188b13ec5c30c7f1ab7e8184031dca3ee057a7242", "train_cache_sha256": "9dfb110a8a940b37c47f6b9212bdc0383a75c53810d2f145c70a05d081c10861", "validation_cache_sha256": "ea3bd0f2625e78723baf6300dca2ffdae91deebdcb21ae748ffd0cc4ba2ef859", "graph_sha256": "a2f35369a5ed4981b8ef75c92df285f7df576a020582f119a1372bfbedbaebee", "validation_panel_sha256": "9a0c030150fee1821246dd54f045fc36c437bab3061e3010740d2942900de3ae", "evaluation_unit": "block", "cache_identity": "SHA-256 of verified mmap manifest" } }, { "scale": "100m", "seed": 43, "variant": "gru", "complete": true, "parameters": 595408, "validation": [ { "step": 0, "presented_tokens": 0, "components": { "bnc_spoken": { "bits_per_byte": 3.5452520671982803, "token_perplexity": 4074.73420074953, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 5.011258447882946, "token_perplexity": 4142.05660518719, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 3.5530831860636027, "token_perplexity": 4085.8675966965293, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 4.219522609329652, "token_perplexity": 4092.459266471993, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 3.861212057914252, "token_perplexity": 4086.158511624914, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 4.461841317730949, "token_perplexity": 4069.550915179262, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 4.045619225940278, "token_perplexity": 4091.7360537654986, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "0ed6229b22b3e978562db4c214d6811fad8e385d95906a473a846e20e30305b9" }, { "step": 500, "presented_tokens": 768000, "components": { "bnc_spoken": { "bits_per_byte": 2.2477766368832075, "token_perplexity": 194.4873508808114, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.1168545813681474, "token_perplexity": 33.727451020144954, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.4164687626007106, "token_perplexity": 285.7930145594125, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.481664923555946, "token_perplexity": 133.15212924542374, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.7763297747383167, "token_perplexity": 395.04474071498805, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.158244686832302, "token_perplexity": 55.71597515355422, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.3782809743285656, "token_perplexity": 132.83630023419408, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "6eb1b0576c607d8b09dee14e7e73153e772c5d6ec5122edacf07b36607a01b98" }, { "step": 1000, "presented_tokens": 1536000, "components": { "bnc_spoken": { "bits_per_byte": 2.1593884252866857, "token_perplexity": 158.08345359249833, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9503048562647707, "token_perplexity": 25.572014857931382, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.269617703670625, "token_perplexity": 202.6726117097814, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.3385289434090004, "token_perplexity": 100.42023651864584, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.5807969848130017, "token_perplexity": 259.27959297606657, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.9121491911125825, "token_perplexity": 35.22841190250966, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.216995013185188, "token_perplexity": 95.35017874561093, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "dc964e55ab9c406984c1713712b52130187763a32986dcd8e1290b948291f928" }, { "step": 1500, "presented_tokens": 2304000, "components": { "bnc_spoken": { "bits_per_byte": 2.053817009544971, "token_perplexity": 123.41955113319841, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9112448269155147, "token_perplexity": 23.964629545655022, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.182143616809937, "token_perplexity": 165.1536256162789, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.268311205509916, "token_perplexity": 87.44070314442126, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.4796790732522025, "token_perplexity": 208.54258621006707, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8354839862869885, "token_perplexity": 30.540232814844803, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.134612223267534, "token_perplexity": 80.49531219176237, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "ac66be70383dda48420330e8d82dba10dd7281023c38914a5843ca76d4cc97a0" }, { "step": 2000, "presented_tokens": 3072000, "components": { "bnc_spoken": { "bits_per_byte": 2.008847681251396, "token_perplexity": 111.0688019823392, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8568668426953931, "token_perplexity": 21.8937279731526, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.1243205760840027, "token_perplexity": 144.2506915961799, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.209768984586973, "token_perplexity": 77.91130651348541, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.402558004377515, "token_perplexity": 176.6301939734512, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7786783782237516, "token_perplexity": 27.47373247629749, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0763661507123747, "token_perplexity": 71.41160617047377, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "5b35187d846a821ca4aa40d081fdf042272f7bccb1626437cca87143870f4b29" }, { "step": 2500, "presented_tokens": 3840000, "components": { "bnc_spoken": { "bits_per_byte": 1.9729621895228209, "token_perplexity": 102.10575100430088, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8300259054371482, "token_perplexity": 20.938494337771413, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.082428789413165, "token_perplexity": 130.7795717857363, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.178442916413127, "token_perplexity": 73.246154852137, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.354295663022488, "token_perplexity": 159.19379402614956, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.752586447051556, "token_perplexity": 26.17036494355966, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.040606438454328, "token_perplexity": 66.35027356287273, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "406b9f9490fc2781b64b7bb4d37a13719de97eef2e0b1e249865d420612f4888" }, { "step": 3000, "presented_tokens": 4608000, "components": { "bnc_spoken": { "bits_per_byte": 1.9420647829764177, "token_perplexity": 94.97026375642014, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7821114774787659, "token_perplexity": 19.335702584945615, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.051604734125216, "token_perplexity": 121.6776865222513, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1548422682219686, "token_perplexity": 69.91691664990833, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.3029079514167554, "token_perplexity": 142.51619666047586, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7218857924384139, "token_perplexity": 24.715734502679577, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.0050481072376334, "token_perplexity": 61.673191954162895, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "8bd9289647471a267d1de11b72ae4c25b0c688648aab42a1587c20e0e8246114" }, { "step": 3500, "presented_tokens": 5376000, "components": { "bnc_spoken": { "bits_per_byte": 1.922835988645317, "token_perplexity": 90.78354353338176, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7688419046046326, "token_perplexity": 18.91392846353716, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0245888676626396, "token_perplexity": 114.22273247338244, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1342251264199996, "token_perplexity": 67.13262627923451, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.273366807117434, "token_perplexity": 133.73188809900444, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7056919064081508, "token_perplexity": 23.9813136546298, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9835807698059504, "token_perplexity": 59.01066382437522, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "fe4090c6632487ff13dad1aca3e36e5fdd21a9c91c5663e6beff5d503652d20e" }, { "step": 4000, "presented_tokens": 6144000, "components": { "bnc_spoken": { "bits_per_byte": 1.9123535672925154, "token_perplexity": 88.57944852522384, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7731626201106436, "token_perplexity": 19.05024265305725, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.008439510133103, "token_perplexity": 109.98631560560621, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.114792475254962, "token_perplexity": 64.60988258475739, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.255817029926372, "token_perplexity": 128.7718743120292, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.683984784093226, "token_perplexity": 23.03097436015149, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9696584615977397, "token_perplexity": 57.345683897684246, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "4eab24c5a85b07b2587c286853e5275f49c77f0b9247523bf4c3f46688041724" }, { "step": 4500, "presented_tokens": 6912000, "components": { "bnc_spoken": { "bits_per_byte": 1.8899293844374176, "token_perplexity": 84.04243407684751, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.773230731369177, "token_perplexity": 19.052399341113546, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9864222479672364, "token_perplexity": 104.46257793145082, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0967583931787726, "token_perplexity": 62.35359036015844, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.221067521620586, "token_perplexity": 119.48695389881904, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6762169268299083, "token_perplexity": 22.700125594665575, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9511052674224176, "token_perplexity": 55.199677711648896, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "76f5a5026e6f64f86bcff8994193102434aeb50ca0399159375a652528f9b184" }, { "step": 5000, "presented_tokens": 7680000, "components": { "bnc_spoken": { "bits_per_byte": 1.8785563618843377, "token_perplexity": 81.83094610805439, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7420156943775909, "token_perplexity": 18.089147643438814, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.972262369725407, "token_perplexity": 101.05758503903195, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0857757115932927, "token_perplexity": 61.018299466395725, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1993581503588606, "token_perplexity": 114.02921760808348, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6630780173976547, "token_perplexity": 22.151295321192272, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9345207745537971, "token_perplexity": 53.34945582357948, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "ee68c9f601ff32c372e54be125198f0f709f024d3d747e55c215900c3c558b83" }, { "step": 5500, "presented_tokens": 8448000, "components": { "bnc_spoken": { "bits_per_byte": 1.866300717142902, "token_perplexity": 79.51292980676739, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7517003250280936, "token_perplexity": 18.382672211873153, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.965141092389676, "token_perplexity": 99.38732200459043, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.074781889400632, "token_perplexity": 59.71029241013222, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.183914653855593, "token_perplexity": 110.29914549401903, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6548987169493383, "token_perplexity": 21.81635668666342, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.926456396973691, "token_perplexity": 52.47230712990326, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "eb57ee121ef5d62c8a4e07d6ea99b2543a4f8ee0d3cf9be0ecf7c55a160e12e4" }, { "step": 6000, "presented_tokens": 9216000, "components": { "bnc_spoken": { "bits_per_byte": 1.8608001011943696, "token_perplexity": 78.4940119087517, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.741993347752358, "token_perplexity": 18.088475804272882, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.956933802471896, "token_perplexity": 97.49655259736801, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0621328048838596, "token_perplexity": 58.23999952641621, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1665993426152674, "token_perplexity": 106.26187627909378, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6491196544050502, "token_perplexity": 21.582764481028175, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9166282169535749, "token_perplexity": 51.42278719352694, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "23c3c8972b3eed6befefa54c58fb2db47fa3060347fcc228c4cd0ff23e2c740c" }, { "step": 6500, "presented_tokens": 9984000, "components": { "bnc_spoken": { "bits_per_byte": 1.8503132073368764, "token_perplexity": 76.5874860538228, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7420572474453133, "token_perplexity": 18.090396980220746, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.946453139162323, "token_perplexity": 95.13425430034744, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0627560060458867, "token_perplexity": 58.311583303865866, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.155671627145105, "token_perplexity": 103.79035659119249, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.639040367313194, "token_perplexity": 21.181324618344036, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.909324207014284, "token_perplexity": 50.656435936943716, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "7abef51577da511cbef92e2dfc51785ea14e9b4b38750c6d6de4e7892a55f0b0" }, { "step": 7000, "presented_tokens": 10752000, "components": { "bnc_spoken": { "bits_per_byte": 1.8436722806388055, "token_perplexity": 75.40417913170533, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7287971074470372, "token_perplexity": 17.696064430657945, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9387871825551954, "token_perplexity": 93.44270240209177, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.048120574610175, "token_perplexity": 56.653487720462955, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.142365695252218, "token_perplexity": 100.85843600170001, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.637147403280242, "token_perplexity": 21.106768339439682, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8998726762268994, "token_perplexity": 49.68168919113374, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "37f6fde1f7f548aa62a8c55671d4afcb5655a7706364ed03d76b1b308068ee35" }, { "step": 7500, "presented_tokens": 11520000, "components": { "bnc_spoken": { "bits_per_byte": 1.8326339452241318, "token_perplexity": 73.47763607401566, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7085731404034594, "token_perplexity": 17.11112916161922, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9313294212718655, "token_perplexity": 91.82595783425819, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.042105751437434, "token_perplexity": 55.98579614797049, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.128010437284126, "token_perplexity": 97.7881073687043, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6331870538862032, "token_perplexity": 20.951633795438603, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8895685133720115, "token_perplexity": 48.64036788856524, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "1b303ec6896c4bee5af4743fbe349cc623f18ca2c0c119b83de0649939182a8d" }, { "step": 8000, "presented_tokens": 12288000, "components": { "bnc_spoken": { "bits_per_byte": 1.8305596054920776, "token_perplexity": 73.12112989259589, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7018235854423127, "token_perplexity": 16.9202478352162, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9260587459554785, "token_perplexity": 90.7002440859331, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.028424474403552, "token_perplexity": 54.49622866988035, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1188644238044785, "token_perplexity": 95.88086762856109, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6242728051916184, "token_perplexity": 20.606604392351837, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.882135263407393, "token_perplexity": 47.90275423397646, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "7328577151700b26fcb1f680f7b3fc1ee2f71a0c24054ec9da3f81b2a7dcd7f4" }, { "step": 8500, "presented_tokens": 13056000, "components": { "bnc_spoken": { "bits_per_byte": 1.8239317118156722, "token_perplexity": 71.99357952909683, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7006888565372307, "token_perplexity": 16.88836677890288, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9196901084246423, "token_perplexity": 89.35842328547494, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.02963239254007, "token_perplexity": 54.626131701006784, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1102230207745647, "token_perplexity": 94.11304434951862, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.613630359327568, "token_perplexity": 20.202117519392544, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.87659109486245, "token_perplexity": 47.359889443903036, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "90fa351237736ae6aee4759627163fd116933f19f4cb99c9b6a9b62d08085ce1" }, { "step": 9000, "presented_tokens": 13824000, "components": { "bnc_spoken": { "bits_per_byte": 1.8178290403327955, "token_perplexity": 70.9707623089173, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.6928971665962773, "token_perplexity": 16.671069847888838, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9166331156427965, "token_perplexity": 88.72140882004994, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.022042845628988, "token_perplexity": 53.815039571745494, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1016974348565616, "token_perplexity": 92.4008593663618, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6133743025262144, "token_perplexity": 20.19248402502288, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.871061028274313, "token_perplexity": 46.82453416526373, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "54f723d3e6b638aaa65470378d27a87164ec08f75433fd4d366a34992004c78b" }, { "step": 9500, "presented_tokens": 14592000, "components": { "bnc_spoken": { "bits_per_byte": 1.8131030772906143, "token_perplexity": 70.18867808523302, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.6968071210959257, "token_perplexity": 16.779760053529607, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.913144064935228, "token_perplexity": 87.99990969720669, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.020218967866861, "token_perplexity": 53.62192403179536, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1010135408045816, "token_perplexity": 92.26487076976787, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6071865896420607, "token_perplexity": 19.961078544284167, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8686821719397542, "token_perplexity": 46.59610712387429, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "58f7a9597b80493515cf1ca190b1dc87cd39bd1e5bc6003969ae677fe3074bbf" }, { "step": 10000, "presented_tokens": 15360000, "components": { "bnc_spoken": { "bits_per_byte": 1.813228684764746, "token_perplexity": 70.20935254715374, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.690763114317734, "token_perplexity": 16.612044046997365, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9085194529334026, "token_perplexity": 87.05262295927123, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0145020030824425, "token_perplexity": 53.02108030720106, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0963291448776933, "token_perplexity": 91.33876916948874, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.603819587346238, "token_perplexity": 19.836276869328337, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8647742154649567, "token_perplexity": 46.223266405035446, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "11942c362c67ab7c0b7f0dba2a4ae088152c9d84a777afee7fcc3f2b8797b26a" }, { "step": 10500, "presented_tokens": 16128000, "components": { "bnc_spoken": { "bits_per_byte": 1.8092635630895597, "token_perplexity": 69.55963897010894, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.695324630960683, "token_perplexity": 16.73846614814413, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9040932458978308, "token_perplexity": 86.15552852200628, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0123881094885285, "token_perplexity": 52.800622503481776, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0924801775840653, "token_perplexity": 90.58479213203118, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6041944765581968, "token_perplexity": 19.850133861120717, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8628421012029304, "token_perplexity": 46.04003557449295, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "c22b7b18a5e0fb2737b047cbd5358925b0399f0c6c9026a3e07d68b384b00af8" }, { "step": 11000, "presented_tokens": 16896000, "components": { "bnc_spoken": { "bits_per_byte": 1.8090068746852874, "token_perplexity": 69.51778654769944, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.6851949561738029, "token_perplexity": 16.459016392707934, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.902860120877148, "token_perplexity": 85.90725194953605, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0061404259851923, "token_perplexity": 52.15439568496395, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0874984944416024, "token_perplexity": 89.61815989880708, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6002667433555522, "token_perplexity": 19.70543275059614, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8587330294679625, "token_perplexity": 45.65276571038168, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "ea28e4041b3a3b18fa92ae3dc31960ba5a6a019f8375bf68420aaa8690f2ed9a" }, { "step": 11500, "presented_tokens": 17664000, "components": { "bnc_spoken": { "bits_per_byte": 1.8066129796566333, "token_perplexity": 69.12867825807777, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.690194555206091, "token_perplexity": 16.596353544585078, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9010710506301844, "token_perplexity": 85.54831320371002, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.003767119445483, "token_perplexity": 51.91099175501742, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.085104876503548, "token_perplexity": 89.15738436134835, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.5998163963278165, "token_perplexity": 19.688909125361167, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.857747010433656, "token_perplexity": 45.560321519264335, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "289107e6d06f085fa0b24377620917aefdb9e03a4d445ae588ee3029f8649a7c" }, { "step": 12000, "presented_tokens": 18432000, "components": { "bnc_spoken": { "bits_per_byte": 1.8062370199589437, "token_perplexity": 69.06776724026786, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.6838159015512233, "token_perplexity": 16.42133464286397, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.8997863879448906, "token_perplexity": 85.29149851334682, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.004144639459404, "token_perplexity": 51.94963371358493, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.084051493118947, "token_perplexity": 88.95535769223054, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.5988740015846967, "token_perplexity": 19.654376660192977, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8563063669516588, "token_perplexity": 45.42559047519484, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "fe3f99f1589494e6ef9760fb0fc46a32cf0129ce0c69407759fbdf66eaa30edb" } ], "source_commit": "f33de49b3b39f888830d0c030ec5eb0de1ae35c4", "protocol": { "steps": 12000, "batch": 16, "sequence": 96, "warmup": 16, "learning_rate": 0.002, "final_learning_rate": 0.0002, "lr_warmup_updates": 100, "weight_decay": 0.01, "eval_tokens": 49152, "threads": 4, "tokenizer_sha256": "46a2ef1bf8dce00e801c5ac188b13ec5c30c7f1ab7e8184031dca3ee057a7242", "train_cache_sha256": "9dfb110a8a940b37c47f6b9212bdc0383a75c53810d2f145c70a05d081c10861", "validation_cache_sha256": "ea3bd0f2625e78723baf6300dca2ffdae91deebdcb21ae748ffd0cc4ba2ef859", "graph_sha256": "a2f35369a5ed4981b8ef75c92df285f7df576a020582f119a1372bfbedbaebee", "validation_panel_sha256": "9a0c030150fee1821246dd54f045fc36c437bab3061e3010740d2942900de3ae", "evaluation_unit": "block", "cache_identity": "SHA-256 of verified mmap manifest" } }, { "scale": "100m", "seed": 43, "variant": "transformer", "complete": true, "parameters": 607468, "validation": [ { "step": 0, "presented_tokens": 0, "components": { "bnc_spoken": { "bits_per_byte": 3.620318122014462, "token_perplexity": 4858.904843572235, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 5.117717416353461, "token_perplexity": 4943.790040721126, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 3.597350530048216, "token_perplexity": 4531.862787677862, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 4.309159989521228, "token_perplexity": 4883.323707516218, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 3.914240346163621, "token_perplexity": 4580.485559225271, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 4.577161867072917, "token_perplexity": 5044.73853515854, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 4.123631780135422, "token_perplexity": 4803.489102471148, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "80ab9d5c4de20b2611e057ac6c70cd8e31f8a93d44aadb436b2e05b4085d41fa" }, { "step": 500, "presented_tokens": 768000, "components": { "bnc_spoken": { "bits_per_byte": 2.232708940167786, "token_perplexity": 187.73621451370673, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 2.0732666449692587, "token_perplexity": 31.37046582984555, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.391667248819904, "token_perplexity": 269.6769549258078, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.4727076221220483, "token_perplexity": 130.8219175622043, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.7752470673252017, "token_perplexity": 394.12469669342926, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 2.0238516130839033, "token_perplexity": 43.37687326801043, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.3429209690287545, "token_perplexity": 123.5229184097892, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "d317ac4fae6818b7c0c8ebad9188a5e1003309ac87c8d97d829d7607c9d97e35" }, { "step": 1000, "presented_tokens": 1536000, "components": { "bnc_spoken": { "bits_per_byte": 2.1369948548593505, "token_perplexity": 149.99722951577866, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.990216378328788, "token_perplexity": 27.32585748458344, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.256355513905386, "token_perplexity": 196.47877131320513, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.3502858986110877, "token_perplexity": 102.77450577651975, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.569440905345844, "token_perplexity": 253.01555393411155, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.9194439876076677, "token_perplexity": 35.710375760409995, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.216597504509137, "token_perplexity": 95.27229299567534, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "592ebd0dc4216e79207b5e5960e9826dd52f5298980dfab5afc58b59d060d359" }, { "step": 1500, "presented_tokens": 2304000, "components": { "bnc_spoken": { "bits_per_byte": 2.0382657834359756, "token_perplexity": 119.0003625596211, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9365851647548087, "token_perplexity": 24.99550003495413, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.164586843451081, "token_perplexity": 158.50528632077553, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.277862320490086, "token_perplexity": 89.10243260643999, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.469993887377934, "token_perplexity": 204.23792637624857, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.8316961663259927, "token_perplexity": 30.325506810874412, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.13100264996133, "token_perplexity": 79.900221078563, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "b861242bb6681fa1615ba97e9cdf46aa44bdeb32c67f32a709769f5c46d77fa2" }, { "step": 2000, "presented_tokens": 3072000, "components": { "bnc_spoken": { "bits_per_byte": 2.006064456675325, "token_perplexity": 110.34634539085314, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.9606607743820808, "token_perplexity": 26.015970719552584, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.11895623853966, "token_perplexity": 142.45106359748348, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.2314780920577557, "token_perplexity": 81.31747013097169, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.4088279165523576, "token_perplexity": 179.03134345189622, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7893687184320046, "token_perplexity": 28.026311910374528, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.094983673296122, "token_perplexity": 74.19769736076148, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "2354486040f36d4f185832b73aac34c698535fdcb26b1255f5a2eec543f1c01d" }, { "step": 2500, "presented_tokens": 3840000, "components": { "bnc_spoken": { "bits_per_byte": 1.9641422164353395, "token_perplexity": 100.01586658702672, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.921072138638812, "token_perplexity": 24.359268887701127, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.087037197700721, "token_perplexity": 132.19767194105918, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.202023080780275, "token_perplexity": 76.73082355226919, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.3669453348508602, "token_perplexity": 163.5901388317475, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.764099562635962, "token_perplexity": 26.73767714147127, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.059788022849991, "token_perplexity": 69.01888447402641, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "b07f82ce7389412cc62151c0e59229bfc299ccb66d52a46e9223da45cebcaba8" }, { "step": 3000, "presented_tokens": 4608000, "components": { "bnc_spoken": { "bits_per_byte": 1.9332578413217898, "token_perplexity": 93.02926981760842, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8631024443279698, "token_perplexity": 22.121811459088786, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0512873103745486, "token_perplexity": 121.58732967837048, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.168939183742235, "token_perplexity": 71.88685288767454, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.3112451159210083, "token_perplexity": 145.09812778278814, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7280933602215902, "token_perplexity": 25.003184640957794, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.01887298489746, "token_perplexity": 63.45110716414016, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "94aa9e7fd04a15f604471451fefdbc5db3f7631e7293291802c1a419f45942d9" }, { "step": 3500, "presented_tokens": 5376000, "components": { "bnc_spoken": { "bits_per_byte": 1.9122195375280207, "token_perplexity": 88.55161593000611, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8834834485260485, "token_perplexity": 22.884006378683925, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0252196576692376, "token_perplexity": 114.39147709600897, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1398539529271603, "token_perplexity": 67.88159062974063, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2845445241152613, "token_perplexity": 136.99012870877795, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.7120949153538887, "token_perplexity": 24.26905602492292, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 2.000948215524234, "token_perplexity": 61.15557650011892, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "8354ff490c5e0aa0088223a678b09528477a988557fc76a1bc2fce8db34c69a1" }, { "step": 4000, "presented_tokens": 6144000, "components": { "bnc_spoken": { "bits_per_byte": 1.897673189483414, "token_perplexity": 85.5823231560627, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8617844024922976, "token_perplexity": 22.073403406725244, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 2.0072626949893575, "token_perplexity": 109.68381866581278, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.126348716197748, "token_perplexity": 66.098452823543, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.266229210223931, "token_perplexity": 131.69197872299893, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.693907397872799, "token_perplexity": 23.460621963389997, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9837493549597491, "token_perplexity": 59.0311184908225, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "d12f3117d9ef047fef701ddc64e8601e91058301a7247c29d4a81452bbf39339" }, { "step": 4500, "presented_tokens": 6912000, "components": { "bnc_spoken": { "bits_per_byte": 1.8890517654168328, "token_perplexity": 83.86967311801288, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.8680299883818552, "token_perplexity": 22.30372881906316, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.984019384939567, "token_perplexity": 103.87678893886664, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.1019071614819347, "token_perplexity": 62.989607218981206, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2306661365692397, "token_perplexity": 121.9825995504393, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6946179474720007, "token_perplexity": 23.491694418545496, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9685342112815791, "token_perplexity": 57.21330178545627, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "baba3676e1e89ce6cf27cbbd5cc3e6be7b9da46d90fc3fb275a584a9b6521eab" }, { "step": 5000, "presented_tokens": 7680000, "components": { "bnc_spoken": { "bits_per_byte": 1.8733620775533466, "token_perplexity": 80.84036727435937, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7251144015397462, "token_perplexity": 17.588080627330264, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9752767174257517, "token_perplexity": 101.77301482277869, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.090194183753458, "token_perplexity": 61.552031134802256, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.2179419368713242, "token_perplexity": 118.68537113159897, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.67190227608229, "token_perplexity": 22.518413411031762, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.9371112861844775, "token_perplexity": 53.634321266623, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "eac21192606b3198791cc7415f51e056e02163ede57360fe87ed442997735db2" }, { "step": 5500, "presented_tokens": 8448000, "components": { "bnc_spoken": { "bits_per_byte": 1.8542774432495883, "token_perplexity": 77.30268185878117, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7579307322470317, "token_perplexity": 18.574018023518537, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9661671673856835, "token_perplexity": 99.6262701315846, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0817149579131713, "token_perplexity": 60.53186078747385, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1920092983342645, "token_perplexity": 112.2387725714572, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6661220376032086, "token_perplexity": 22.27725534232661, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.929421235121002, "token_perplexity": 52.793098469488065, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "8caff2c3fa9936a0e7cb07926f91ca82f2b140a214ae4874cec31a17c593793f" }, { "step": 6000, "presented_tokens": 9216000, "components": { "bnc_spoken": { "bits_per_byte": 1.8512230005870396, "token_perplexity": 76.75103642724858, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7954113908837381, "token_perplexity": 19.767878942134715, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9531291697246882, "token_perplexity": 96.63229749840131, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.065942506979824, "token_perplexity": 58.67897698273794, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1668775368645417, "token_perplexity": 106.32555776104859, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6623406632847946, "token_perplexity": 22.120891286273046, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.923937806183488, "token_perplexity": 52.20133084721645, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "ae391f5e2a6db2ab15b7b8b8dea8c28869aed275f7b72a2dda70f3724e9861e8" }, { "step": 6500, "presented_tokens": 9984000, "components": { "bnc_spoken": { "bits_per_byte": 1.8355169733849204, "token_perplexity": 73.97601593439516, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.77052084892059, "token_perplexity": 18.966781197286082, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9420506650485947, "token_perplexity": 94.15910671944171, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.063214679840698, "token_perplexity": 58.36432490689307, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1618007089085847, "token_perplexity": 105.1694040366203, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6505584336230759, "token_perplexity": 21.640685679269254, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.912394530993916, "token_perplexity": 50.97717913691798, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "e0c1b8fbf3e157a65d04743e5a223ff1b18ef1f44342d6e5da535571dffc9e18" }, { "step": 7000, "presented_tokens": 10752000, "components": { "bnc_spoken": { "bits_per_byte": 1.833408842807471, "token_perplexity": 73.61125925106425, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.73363427542777, "token_perplexity": 17.838907160454266, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9347201577199489, "token_perplexity": 92.55752763951189, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.04621397058685, "token_perplexity": 56.440982501194156, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1511254409415663, "token_perplexity": 102.77915577554805, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6406828672346707, "token_perplexity": 21.246229479634895, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.899560472285908, "token_perplexity": 49.64981327378826, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "adefcfaabe2dd26844cdce637daa78d11860869ea2fd4d502c909be74c1f7c39" }, { "step": 7500, "presented_tokens": 11520000, "components": { "bnc_spoken": { "bits_per_byte": 1.8184872158709513, "token_perplexity": 71.08037076826137, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7012550902101897, "token_perplexity": 16.90426802114644, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9186604319842084, "token_perplexity": 89.14335067899398, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.04065189889276, "token_perplexity": 55.82559178523507, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1304224398242955, "token_perplexity": 98.29737921831446, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6372360433650466, "token_perplexity": 21.110253653900244, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.884289701814757, "token_perplexity": 48.11538395769949, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "6698f25a49559a1fcf322102263190d80152047e68e3555f760843d354d7c484" }, { "step": 8000, "presented_tokens": 12288000, "components": { "bnc_spoken": { "bits_per_byte": 1.8107318990904098, "token_perplexity": 69.79953236660327, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7116660534223629, "token_perplexity": 17.19931654329669, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9162105281230903, "token_perplexity": 88.63370826320669, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0284964333326423, "token_perplexity": 54.50395868194363, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.116998265601944, "token_perplexity": 95.49630644371177, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6264704383303152, "token_perplexity": 20.69113326199393, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8778413713658229, "token_perplexity": 47.48177221122268, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "0a966366d8427b0ff774a0f7126df187f352e802439332e8adfa5e3e9d470f08" }, { "step": 8500, "presented_tokens": 13056000, "components": { "bnc_spoken": { "bits_per_byte": 1.8070400625456098, "token_perplexity": 69.1979371607475, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.722309164898183, "token_perplexity": 17.506268206577364, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9070359228643492, "token_perplexity": 86.75090823236211, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0248230078233616, "token_perplexity": 54.110747185275244, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.1038116779383658, "token_perplexity": 92.82253396526875, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6245061932499798, "token_perplexity": 20.615564921944273, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8736469444577755, "token_perplexity": 47.07411433991421, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "5b0368b9e112ea6df02c688877d2ea49c90963b6176a0512ea0f3974a2c028cb" }, { "step": 9000, "presented_tokens": 13824000, "components": { "bnc_spoken": { "bits_per_byte": 1.798851008138649, "token_perplexity": 67.88194933902649, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.677867790260892, "token_perplexity": 16.259792472610304, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9036009089452584, "token_perplexity": 86.05631579225016, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.009563688210489, "token_perplexity": 52.50749374156916, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0973533443497727, "token_perplexity": 91.54045536761149, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.617442907917333, "token_perplexity": 20.346099963265576, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8608173135736275, "token_perplexity": 45.84879550181453, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "3de9325f141497880ba7c269f100dcf88125803ff438f3acd18fcc5661f2f225" }, { "step": 9500, "presented_tokens": 14592000, "components": { "bnc_spoken": { "bits_per_byte": 1.7902466431423616, "token_perplexity": 66.52617522961296, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7389236723098942, "token_perplexity": 17.996424411298513, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.9008149541057722, "token_perplexity": 85.49705575476645, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0164459024830705, "token_perplexity": 53.224621756116285, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0922641342916393, "token_perplexity": 90.54265621165948, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6076744425900495, "token_perplexity": 19.979226365466893, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.865616258870634, "token_perplexity": 46.30334904259768, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "2126311ca6fa47bf38f08fcb5c36952a8c05d70564780f42ffab216db76b7029" }, { "step": 10000, "presented_tokens": 15360000, "components": { "bnc_spoken": { "bits_per_byte": 1.790140600294438, "token_perplexity": 66.50963628991266, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7360129616144797, "token_perplexity": 17.90957267114524, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.8936552053845777, "token_perplexity": 84.07640443930377, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 2.0082797693790893, "token_perplexity": 52.37478251052514, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0861084253000493, "token_perplexity": 89.35028009823108, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6076250493289632, "token_perplexity": 19.97738821695819, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8614582228426295, "token_perplexity": 45.9092429444838, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "3b332cb4dce55ef9fad7a81173087c3d486dc803a61542e809bd888dcc95f382" }, { "step": 10500, "presented_tokens": 16128000, "components": { "bnc_spoken": { "bits_per_byte": 1.7854454085611806, "token_perplexity": 65.78145875361369, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7204547816430538, "token_perplexity": 17.452395683323648, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.8897569295109977, "token_perplexity": 83.31285153594118, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 1.9973137311368685, "token_perplexity": 51.25486737099764, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.081443157800916, "token_perplexity": 88.45707732822555, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6037508571574066, "token_perplexity": 19.83373745184484, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.854577586316588, "token_perplexity": 45.26443825506643, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "decd55bb878ca4fc8b3e0c8810d3d3b8e507054659d9fe8ac4e9428f2881c4a0" }, { "step": 11000, "presented_tokens": 16896000, "components": { "bnc_spoken": { "bits_per_byte": 1.7851218025893723, "token_perplexity": 65.73156536612923, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.7077332110025976, "token_perplexity": 17.08725865236389, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.8883434385821636, "token_perplexity": 83.03770826720344, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 1.9960777011348456, "token_perplexity": 51.130148137069234, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0718400478966177, "token_perplexity": 86.64649258163664, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.6009111666215359, "token_perplexity": 19.729101324154083, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.85023061007166, "token_perplexity": 44.861746548605346, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "291984387d38ba6268c04528b7b78c052581a28c14d4286853ee667ce5d74299" }, { "step": 11500, "presented_tokens": 17664000, "components": { "bnc_spoken": { "bits_per_byte": 1.7843069201785626, "token_perplexity": 65.60609481472358, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.707612074138603, "token_perplexity": 17.08381873242172, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.8857678598306138, "token_perplexity": 82.53869286149231, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 1.9925568941004492, "token_perplexity": 50.776548800064965, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.070542022853533, "token_perplexity": 86.40462112303331, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.5993738418679742, "token_perplexity": 19.672684914333384, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.848549269867187, "token_perplexity": 44.706954459035636, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "19903a2597237f31bb209dc1916af646d3ce3bce6021ea63e532b15c5ed4e4dd" }, { "step": 12000, "presented_tokens": 18432000, "components": { "bnc_spoken": { "bits_per_byte": 1.7804654676273395, "token_perplexity": 65.01782984785032, "bytes": 27711, "tokens": 8192 }, "childes": { "bits_per_byte": 1.698509304109016, "token_perplexity": 16.827299028180825, "bytes": 19643, "tokens": 8192 }, "gutenberg": { "bits_per_byte": 1.8842099346643595, "token_perplexity": 82.23830340189375, "bytes": 27659, "tokens": 8192 }, "open_subtitles": { "bits_per_byte": 1.994104273313436, "token_perplexity": 50.93165195429018, "bytes": 23295, "tokens": 8192 }, "simple_wiki": { "bits_per_byte": 2.0687515428555443, "token_perplexity": 86.07209417626522, "bytes": 25452, "tokens": 8192 }, "switchboard": { "bits_per_byte": 1.5995569467145982, "token_perplexity": 19.679395984036343, "bytes": 22015, "tokens": 8192 }, "all": { "bits_per_byte": 1.8462591590768913, "token_perplexity": 44.496974910567424, "bytes": 145775, "tokens": 49152 } }, "artifact_sha256": "d756ce6e38b336a621ab65f748c68b1d4ae8da1fbe43d794774f9e62fb2e1f83" } ], "source_commit": "096a4fcd64a04de02a99f38127fc60fb866bf3d2", "protocol": { "steps": 12000, "batch": 16, "sequence": 96, "warmup": 16, "learning_rate": 0.002, "final_learning_rate": 0.0002, "lr_warmup_updates": 100, "weight_decay": 0.01, "eval_tokens": 49152, "threads": 4, "tokenizer_sha256": "46a2ef1bf8dce00e801c5ac188b13ec5c30c7f1ab7e8184031dca3ee057a7242", "train_cache_sha256": "9dfb110a8a940b37c47f6b9212bdc0383a75c53810d2f145c70a05d081c10861", "validation_cache_sha256": "ea3bd0f2625e78723baf6300dca2ffdae91deebdcb21ae748ffd0cc4ba2ef859", "graph_sha256": "a2f35369a5ed4981b8ef75c92df285f7df576a020582f119a1372bfbedbaebee", "validation_panel_sha256": "9a0c030150fee1821246dd54f045fc36c437bab3061e3010740d2942900de3ae", "evaluation_unit": "block", "cache_identity": "SHA-256 of verified mmap manifest" } } ], "tokenizer_sha256": "46a2ef1bf8dce00e801c5ac188b13ec5c30c7f1ab7e8184031dca3ee057a7242", "validation_panel_sha256": "9a0c030150fee1821246dd54f045fc36c437bab3061e3010740d2942900de3ae", "training_text_tokens": { "10m": 18591514, "100m": 189709779 }, "measurement": "Validation only, fixed 48-block panel; choose an update present for all three models. This snapshot is not a process monitor or a test result.", "exposure": "18,432,000 presented tokens per completed run, sampled with replacement; the 100M-word corpus receives proportionally less coverage.", "planned_continuations": 288 }