{
  "dataset": "WikiText-2 raw",
  "status": "validation progress; test results not included",
  "measurement": "Fixed article prefixes; lower bits per UTF-8 byte is better. Compare models at the same update count and seed.",
  "runs": [
    {
      "id": "flm-s42",
      "variant": "flm",
      "seed": 42,
      "parameters": 600003,
      "protocol": {
        "steps": 6000,
        "batch": 16,
        "sequence": 96,
        "warmup": 16,
        "learning_rate": 0.002,
        "final_learning_rate": 0.0002,
        "lr_warmup_updates": 100,
        "weight_decay": 0.01,
        "eval_tokens": 32768,
        "threads": 4,
        "tokenizer_sha256": "ef7f54d89966a08485006e0f64eaf2b0b99217b65ad79f96979f12adbd9ca49d",
        "train_cache_sha256": "dccfcab2728e223fc1620a00afba24feafe6d47373ec7619632d3e5b80b89789",
        "validation_cache_sha256": "a9fe379a68775dc5808338d69d3061c0dfbf2c1e7ae64b8a1f1f55d57e4e6075",
        "graph_sha256": "a2f35369a5ed4981b8ef75c92df285f7df576a020582f119a1372bfbedbaebee"
      },
      "complete": true,
      "validation": [
        {
          "step": 0,
          "presented_tokens": 0,
          "bits_per_byte": 3.4640037142576174,
          "token_perplexity": 5263.335142199168,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "df8b69c24818e50048df799f86fd6b49261a2581ac32550f9ee365a2e7a00936"
        },
        {
          "step": 500,
          "presented_tokens": 768000,
          "bits_per_byte": 2.287044223128317,
          "token_perplexity": 286.3476295029146,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "6a64488a7e0f5324fde73661bdc2703beed14e497fe3700d36b731136abad15c"
        },
        {
          "step": 1000,
          "presented_tokens": 1536000,
          "bits_per_byte": 2.136728308544109,
          "token_perplexity": 197.43053974360623,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "be5e20a5113be3fbece332c14cc856876685f343154c6d5a894c8f91b0f42c33"
        },
        {
          "step": 1500,
          "presented_tokens": 2304000,
          "bits_per_byte": 2.061640680624221,
          "token_perplexity": 163.96466104350574,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "cefda266db6b0b06b54c0fd3073337ccd390e483a75552513f23c212ce40df48"
        },
        {
          "step": 2000,
          "presented_tokens": 3072000,
          "bits_per_byte": 2.0135569920978464,
          "token_perplexity": 145.57794657875584,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "fb543162eeeafc8d0ee534433035b7b723c10406985053cf468718f710862132"
        },
        {
          "step": 2500,
          "presented_tokens": 3840000,
          "bits_per_byte": 1.9854636563028905,
          "token_perplexity": 135.80503660043757,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "bd1652ff21feaf30e322c08d5532da6e190e4e1283d05d6d106d6bf2c9e3a746"
        },
        {
          "step": 3000,
          "presented_tokens": 4608000,
          "bits_per_byte": 1.9630415020620382,
          "token_perplexity": 128.47792725825516,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "a36c1cecb0f71ff363db5e964da8f5bacc322e013f4bed1053cf8c14ed6c04b7"
        },
        {
          "step": 3500,
          "presented_tokens": 5376000,
          "bits_per_byte": 1.9513447595957074,
          "token_perplexity": 124.81394496971464,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "bc932f1a3e285768ae6ac26767ab844ec4e029fd447fb5b21aaba9222a25d2fd"
        },
        {
          "step": 4000,
          "presented_tokens": 6144000,
          "bits_per_byte": 1.9340352182030975,
          "token_perplexity": 119.58261885439221,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "0276b9ae11ed4562ab016444db8efdc8cf9d0c805a1cb77d3cf3f8cf87c27399"
        },
        {
          "step": 4500,
          "presented_tokens": 6912000,
          "bits_per_byte": 1.926545513274328,
          "token_perplexity": 117.3875735611506,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "516f2fd0883ab64c40b230f32ab8e27d96847fc987f61981bc27db05205c8ba5"
        },
        {
          "step": 5000,
          "presented_tokens": 7680000,
          "bits_per_byte": 1.91739326132379,
          "token_perplexity": 114.75990330448276,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "3179bfc34a98de6d447693ea09db267091e5a7df1c3abb6a38614f9c396be0f7"
        },
        {
          "step": 5500,
          "presented_tokens": 8448000,
          "bits_per_byte": 1.9139342327564128,
          "token_perplexity": 113.78218180995947,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "bf57d5b5c5e97b61e071611a3c5fd5dd00400aab4c8fc2d255af763eed50526d"
        },
        {
          "step": 6000,
          "presented_tokens": 9216000,
          "bits_per_byte": 1.9096990391099853,
          "token_perplexity": 112.5964072527091,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "64e5e682ebff6e9589027843e83dea27d6f454f6e1463fcaa5a0674fce6a5bd4"
        }
      ],
      "source_commit": "14cedb6caaa14468f1aa55f53f214ae6ff1d36b5"
    },
    {
      "id": "flm-s43",
      "variant": "flm",
      "seed": 43,
      "parameters": 600003,
      "protocol": {
        "steps": 6000,
        "batch": 16,
        "sequence": 96,
        "warmup": 16,
        "learning_rate": 0.002,
        "final_learning_rate": 0.0002,
        "lr_warmup_updates": 100,
        "weight_decay": 0.01,
        "eval_tokens": 32768,
        "threads": 4,
        "tokenizer_sha256": "ef7f54d89966a08485006e0f64eaf2b0b99217b65ad79f96979f12adbd9ca49d",
        "train_cache_sha256": "dccfcab2728e223fc1620a00afba24feafe6d47373ec7619632d3e5b80b89789",
        "validation_cache_sha256": "a9fe379a68775dc5808338d69d3061c0dfbf2c1e7ae64b8a1f1f55d57e4e6075",
        "graph_sha256": "a2f35369a5ed4981b8ef75c92df285f7df576a020582f119a1372bfbedbaebee"
      },
      "complete": true,
      "validation": [
        {
          "step": 0,
          "presented_tokens": 0,
          "bits_per_byte": 3.4701015414805423,
          "token_perplexity": 5343.326501008048,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "8bf0034910f05366e693ae837168ca9acefcd8c89d332c3ccde9998bdcbe6b53"
        },
        {
          "step": 500,
          "presented_tokens": 768000,
          "bits_per_byte": 2.295645522914354,
          "token_perplexity": 292.50525658484554,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "c32475c9e2565cf8a629809ea7b933cf714edab9d140abcef77c8884608cde50"
        },
        {
          "step": 1000,
          "presented_tokens": 1536000,
          "bits_per_byte": 2.131158570307363,
          "token_perplexity": 194.72914322028018,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "34935e2ee56765c7627339003cdb038467c5ea14cbd53b7e31635ddd55e44a1d"
        },
        {
          "step": 1500,
          "presented_tokens": 2304000,
          "bits_per_byte": 2.0647620122785635,
          "token_perplexity": 165.23551397485568,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "840c429bc9eda50d0abc5aa6afdddfb440e93781413e5d275f26706e15d715d1"
        },
        {
          "step": 2000,
          "presented_tokens": 3072000,
          "bits_per_byte": 2.020279033844324,
          "token_perplexity": 148.01878989312084,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "d7db71196d56ad254086a5e28e9b74725ee3eba5659d54bb9fb0aa27fbf161c4"
        },
        {
          "step": 2500,
          "presented_tokens": 3840000,
          "bits_per_byte": 1.9868651563892965,
          "token_perplexity": 136.2766536414206,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "ca73543d2ba36673bbdbcce375599f46150e2b8421e9173ec1fa55983e7141ae"
        },
        {
          "step": 3000,
          "presented_tokens": 4608000,
          "bits_per_byte": 1.9678971024319716,
          "token_perplexity": 130.03034874347136,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "d436b7cfc0357058568ac4023a650fe74e8b5446e61bebc39afefef5b1999d9a"
        },
        {
          "step": 3500,
          "presented_tokens": 5376000,
          "bits_per_byte": 1.9472440499845827,
          "token_perplexity": 123.55429803031356,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "2c3612ddcdd1dee6b1dbde5a7e2a4a3c19b6c701adfe5b950b4cdcfcda34b097"
        },
        {
          "step": 4000,
          "presented_tokens": 6144000,
          "bits_per_byte": 1.934413596848371,
          "token_perplexity": 119.6945949724892,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "711caed512b2a64b2f123fa811fda01b998e98923ab047bd36c8019df5c6188e"
        },
        {
          "step": 4500,
          "presented_tokens": 6912000,
          "bits_per_byte": 1.9260197578763922,
          "token_perplexity": 117.23500994641536,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "7310e392a25d74511f3f4ecce340e22279af9f0a48c4e233065e6c5f24008628"
        },
        {
          "step": 5000,
          "presented_tokens": 7680000,
          "bits_per_byte": 1.9187546606277253,
          "token_perplexity": 115.14701351761211,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "a4445cf106cdce00abb6e7f5b3ce24065b01b3e0771033227420983d6f1fa01d"
        },
        {
          "step": 5500,
          "presented_tokens": 8448000,
          "bits_per_byte": 1.9139361356210431,
          "token_perplexity": 113.78271737305981,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "fedc18a9375ee579f2f9dff11871860042a85193eb936d1f2fcfd3a739674032"
        },
        {
          "step": 6000,
          "presented_tokens": 9216000,
          "bits_per_byte": 1.9107256505795487,
          "token_perplexity": 112.88269954949243,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "cf85578d3b8fe6f7a340487c3b1ec9e11621a7d7937a3ad518d80327ff5f1d68"
        }
      ],
      "source_commit": "81d15ad7c4395bfbc59e5034ddf955b59dd90d3d"
    },
    {
      "id": "gru-s42",
      "variant": "gru",
      "seed": 42,
      "parameters": 595408,
      "protocol": {
        "steps": 6000,
        "batch": 16,
        "sequence": 96,
        "warmup": 16,
        "learning_rate": 0.002,
        "final_learning_rate": 0.0002,
        "lr_warmup_updates": 100,
        "weight_decay": 0.01,
        "eval_tokens": 32768,
        "threads": 4,
        "tokenizer_sha256": "ef7f54d89966a08485006e0f64eaf2b0b99217b65ad79f96979f12adbd9ca49d",
        "train_cache_sha256": "dccfcab2728e223fc1620a00afba24feafe6d47373ec7619632d3e5b80b89789",
        "validation_cache_sha256": "a9fe379a68775dc5808338d69d3061c0dfbf2c1e7ae64b8a1f1f55d57e4e6075",
        "graph_sha256": "a2f35369a5ed4981b8ef75c92df285f7df576a020582f119a1372bfbedbaebee"
      },
      "complete": true,
      "validation": [
        {
          "step": 0,
          "presented_tokens": 0,
          "bits_per_byte": 3.361639326081018,
          "token_perplexity": 4085.963231628399,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "3798c0dcb5f3f16c7095cf0b7ad7a8421c538eaeb0b70c24d639a106305b3377"
        },
        {
          "step": 500,
          "presented_tokens": 768000,
          "bits_per_byte": 2.294919142180667,
          "token_perplexity": 291.98016463391684,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "74ec0be78ffd330d1902725ca6ca8dd7bffe10a6ae9890109d19c56a1cc571b3"
        },
        {
          "step": 1000,
          "presented_tokens": 1536000,
          "bits_per_byte": 2.1268626259945087,
          "token_perplexity": 192.67082925705628,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "04f470b67b9c4b7894e025ea67e9fe4d5159556602ce2671b3860f5892158088"
        },
        {
          "step": 1500,
          "presented_tokens": 2304000,
          "bits_per_byte": 2.0200993213824265,
          "token_perplexity": 147.95300502761853,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "675b1049129f78207e09192e3ab7daa67e9bf5849653bcbc4e8ac4d71aec7f2e"
        },
        {
          "step": 2000,
          "presented_tokens": 3072000,
          "bits_per_byte": 1.9590084122929925,
          "token_perplexity": 127.20257727810254,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "f4282d53ea7e44821e81def1b719f89455afc873eb1fddb547a6ce78f68a2554"
        },
        {
          "step": 2500,
          "presented_tokens": 3840000,
          "bits_per_byte": 1.9167577884982638,
          "token_perplexity": 114.57965410581271,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "5bf281556032c2d9ef6ad0849a325e3a1246ebec8f2f031760b9355b2984a092"
        },
        {
          "step": 3000,
          "presented_tokens": 4608000,
          "bits_per_byte": 1.89128749869695,
          "token_perplexity": 107.58349364901774,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "b80691a06deb52d2a4de5ee68ffb5a543be0bf3724e007cfd70bd86422e802ee"
        },
        {
          "step": 3500,
          "presented_tokens": 5376000,
          "bits_per_byte": 1.8722841430992219,
          "token_perplexity": 102.64338992494503,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "b9d50f1da3187c1d9bb83b0ee0399a5bcd308cd60d38f43fb0fe9b30bd4a8dee"
        },
        {
          "step": 4000,
          "presented_tokens": 6144000,
          "bits_per_byte": 1.8556756924883373,
          "token_perplexity": 98.51199059388203,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "6246f6a16225d210d994e2b554898359835c4478c09e939db2c4fb0ddc185fc1"
        },
        {
          "step": 4500,
          "presented_tokens": 6912000,
          "bits_per_byte": 1.846391321717298,
          "token_perplexity": 96.2753734626313,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "9cf76db2e8146d392463340371e183fdfcc280e91e526c56c17caf315b12da70"
        },
        {
          "step": 5000,
          "presented_tokens": 7680000,
          "bits_per_byte": 1.837391148711027,
          "token_perplexity": 94.15570361033524,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "fc1da2ad52cfb2edc1a3379f63533807365c1499438513e72c2dd38defb4eba8"
        },
        {
          "step": 5500,
          "presented_tokens": 8448000,
          "bits_per_byte": 1.8329470303926698,
          "token_perplexity": 93.12632574737808,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "b31c5516c0ce537fec7315987b062dd9d1a99be9e4e9c524b2064aa4d50ed080"
        },
        {
          "step": 6000,
          "presented_tokens": 9216000,
          "bits_per_byte": 1.829255113634344,
          "token_perplexity": 92.27974126867089,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "703f3661f4587e7412b522f3cb9a4d067b90d442eb64f7ea1de613eaae4f4970"
        }
      ],
      "source_commit": "14cedb6caaa14468f1aa55f53f214ae6ff1d36b5"
    },
    {
      "id": "gru-s43",
      "variant": "gru",
      "seed": 43,
      "parameters": 595408,
      "protocol": {
        "steps": 6000,
        "batch": 16,
        "sequence": 96,
        "warmup": 16,
        "learning_rate": 0.002,
        "final_learning_rate": 0.0002,
        "lr_warmup_updates": 100,
        "weight_decay": 0.01,
        "eval_tokens": 32768,
        "threads": 4,
        "tokenizer_sha256": "ef7f54d89966a08485006e0f64eaf2b0b99217b65ad79f96979f12adbd9ca49d",
        "train_cache_sha256": "dccfcab2728e223fc1620a00afba24feafe6d47373ec7619632d3e5b80b89789",
        "validation_cache_sha256": "a9fe379a68775dc5808338d69d3061c0dfbf2c1e7ae64b8a1f1f55d57e4e6075",
        "graph_sha256": "a2f35369a5ed4981b8ef75c92df285f7df576a020582f119a1372bfbedbaebee"
      },
      "complete": true,
      "validation": [
        {
          "step": 0,
          "presented_tokens": 0,
          "bits_per_byte": 3.3628182896504257,
          "token_perplexity": 4097.896398881126,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "93dfb33a569d82fdf2054c53b2ea9ed86dbe2a5f3cc3c27f4225b5db78d049f6"
        },
        {
          "step": 500,
          "presented_tokens": 768000,
          "bits_per_byte": 2.29662709534017,
          "token_perplexity": 293.21632410316124,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "b695303b6c1322464e5fdee94d8150f8d369bcd829be7ccc4d97074ed98c257f"
        },
        {
          "step": 1000,
          "presented_tokens": 1536000,
          "bits_per_byte": 2.117295691685834,
          "token_perplexity": 188.164865952122,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "ccfc01df72ce31fd8c22de34e784a53f45a81e755a1f6cc67f7d8ac82b2ba9d7"
        },
        {
          "step": 1500,
          "presented_tokens": 2304000,
          "bits_per_byte": 2.0272341226675583,
          "token_perplexity": 150.58734073936603,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "6bdb71d55a1e0ecc6d419dfde1d359ed1493396d16c7ab678b68efb9c09bdaf9"
        },
        {
          "step": 2000,
          "presented_tokens": 3072000,
          "bits_per_byte": 1.9714659323879231,
          "token_perplexity": 131.18331448531708,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "1266ca441b78508de9228a00cbca3185e26a283974f0b1a504234d907197c9e8"
        },
        {
          "step": 2500,
          "presented_tokens": 3840000,
          "bits_per_byte": 1.9293455565521314,
          "token_perplexity": 118.2034402603944,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "0e58ca664a4bb270f9f02378abaf946dd5357c620e7f5bb724bbb703e34ff749"
        },
        {
          "step": 3000,
          "presented_tokens": 4608000,
          "bits_per_byte": 1.9027440444134145,
          "token_perplexity": 110.67588913167852,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "10775779a660353d3e3d71c762ef152a11b4624045f5fb8fc60060a1cd23670a"
        },
        {
          "step": 3500,
          "presented_tokens": 5376000,
          "bits_per_byte": 1.880813577365894,
          "token_perplexity": 104.83199893325795,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "f3230b7335337f41ef4fe6fdb9f61247384a70e7f4a48c0ef07c7844aa8ed0b0"
        },
        {
          "step": 4000,
          "presented_tokens": 6144000,
          "bits_per_byte": 1.865249161258249,
          "token_perplexity": 100.87267682367518,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "208bee2899d6be60bbffb69a44782a336fde682ad60640a8e0888c34962c06ac"
        },
        {
          "step": 4500,
          "presented_tokens": 6912000,
          "bits_per_byte": 1.8536435925137766,
          "token_perplexity": 98.01805469837068,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "060fed4100636ece65ac3986254d3d1bed16ae8f7a2aedbbe54c302966d6da93"
        },
        {
          "step": 5000,
          "presented_tokens": 7680000,
          "bits_per_byte": 1.8455138377563387,
          "token_perplexity": 96.06663108175506,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "6893dfb2bfd7e1f7124195bea7149ab5e71cd5d5807aa3d03798929d4b6d4d7a"
        },
        {
          "step": 5500,
          "presented_tokens": 8448000,
          "bits_per_byte": 1.840426577360163,
          "token_perplexity": 94.86532318156308,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "43ef6ebf342d11e34efe28d255928d4e2d7239b071ef9a89588caf3ab95c6ee6"
        },
        {
          "step": 6000,
          "presented_tokens": 9216000,
          "bits_per_byte": 1.8364134450793355,
          "token_perplexity": 93.92826914168603,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "f155634b8e01a411fcc696d52741857f32d12415b3d126677cdc71c7ebf99a32"
        }
      ],
      "source_commit": "7d9e53670632bc1d3baa81e78f384a9a9de42951"
    },
    {
      "id": "transformer-s42",
      "variant": "transformer",
      "seed": 42,
      "parameters": 607468,
      "protocol": {
        "steps": 6000,
        "batch": 16,
        "sequence": 96,
        "warmup": 16,
        "learning_rate": 0.002,
        "final_learning_rate": 0.0002,
        "lr_warmup_updates": 100,
        "weight_decay": 0.01,
        "eval_tokens": 32768,
        "threads": 4,
        "tokenizer_sha256": "ef7f54d89966a08485006e0f64eaf2b0b99217b65ad79f96979f12adbd9ca49d",
        "train_cache_sha256": "dccfcab2728e223fc1620a00afba24feafe6d47373ec7619632d3e5b80b89789",
        "validation_cache_sha256": "a9fe379a68775dc5808338d69d3061c0dfbf2c1e7ae64b8a1f1f55d57e4e6075",
        "graph_sha256": "a2f35369a5ed4981b8ef75c92df285f7df576a020582f119a1372bfbedbaebee"
      },
      "complete": true,
      "validation": [
        {
          "step": 0,
          "presented_tokens": 0,
          "bits_per_byte": 3.420612592695807,
          "token_perplexity": 4727.672922253184,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "5a78a45ccb0b6981ebe4f2455d4ba315efbb7ef0d1dd19f7ddb2e34ee4677b1e"
        },
        {
          "step": 500,
          "presented_tokens": 768000,
          "bits_per_byte": 2.2842899376454637,
          "token_perplexity": 284.40338248410904,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "ae66578e65b3d6aa15d81e5cee7aafe7d4bc2f8ca3aabe2752b29029eab3c384"
        },
        {
          "step": 1000,
          "presented_tokens": 1536000,
          "bits_per_byte": 2.1141070839399876,
          "token_perplexity": 186.68658995391604,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "b89d5c55b7843f4d59ca2f01dbb7fdb8c6ed65e25f5aff7ebb35c9a4ba345e2b"
        },
        {
          "step": 1500,
          "presented_tokens": 2304000,
          "bits_per_byte": 2.024381210890134,
          "token_perplexity": 149.52839724122026,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "12459cdc928f4613f26a1a6cb0124fa977ab6e5f2399654136092d406edcf1da"
        },
        {
          "step": 2000,
          "presented_tokens": 3072000,
          "bits_per_byte": 1.9624333137324739,
          "token_perplexity": 128.28478938683926,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "c60befe9e247a252cd9e2feecf70fa88c093d6e283daf458c8dc52187d38a1c4"
        },
        {
          "step": 2500,
          "presented_tokens": 3840000,
          "bits_per_byte": 1.9197967715584685,
          "token_perplexity": 115.44421705627148,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "b1faae15b35fb989475e300e493ed037f709c925d15f76d374d76ef1e7fe9c8f"
        },
        {
          "step": 3000,
          "presented_tokens": 4608000,
          "bits_per_byte": 1.8869946140654208,
          "token_perplexity": 106.44712687942048,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "f69baea1d721bab6b421bd30e8cb340ea5074eebc91c7b17bf6a3124954d206b"
        },
        {
          "step": 3500,
          "presented_tokens": 5376000,
          "bits_per_byte": 1.869146437923502,
          "token_perplexity": 101.84981795409341,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "82fe815e16a873131661c1fbb72dcc39c4038587c30adc1c0b895c10fb3cb08b"
        },
        {
          "step": 4000,
          "presented_tokens": 6144000,
          "bits_per_byte": 1.8451799292532582,
          "token_perplexity": 95.98731738863839,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "cbce9a943dd763c0f8854c02d55cd8c7befd0348eda48f1b963dc8db72740277"
        },
        {
          "step": 4500,
          "presented_tokens": 6912000,
          "bits_per_byte": 1.8345739954648304,
          "token_perplexity": 93.50186242658268,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "be50ff6802d42f534cc5a1d1a511ce713716fc7e61b74ba2a2963bfbd5e9e008"
        },
        {
          "step": 5000,
          "presented_tokens": 7680000,
          "bits_per_byte": 1.8184151399211124,
          "token_perplexity": 89.83826462284816,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "064c342f34e6098993ba9e598b488680ade43cb4ca75df1a5f141a4670efeb24"
        },
        {
          "step": 5500,
          "presented_tokens": 8448000,
          "bits_per_byte": 1.815012692020713,
          "token_perplexity": 89.08533561924266,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "0283fb0bc4a07668914525713880e4f29e793b39e1ad048d4f612de5dbce9715"
        },
        {
          "step": 6000,
          "presented_tokens": 9216000,
          "bits_per_byte": 1.8112767388558602,
          "token_perplexity": 88.26587146796328,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "c6d9b2ad482dca8f2b3f06b7203e1ee6ce23d884817e5bd2b224f5088ee052f8"
        }
      ],
      "source_commit": "14cedb6caaa14468f1aa55f53f214ae6ff1d36b5"
    },
    {
      "id": "transformer-s43",
      "variant": "transformer",
      "seed": 43,
      "parameters": 607468,
      "protocol": {
        "steps": 6000,
        "batch": 16,
        "sequence": 96,
        "warmup": 16,
        "learning_rate": 0.002,
        "final_learning_rate": 0.0002,
        "lr_warmup_updates": 100,
        "weight_decay": 0.01,
        "eval_tokens": 32768,
        "threads": 4,
        "tokenizer_sha256": "ef7f54d89966a08485006e0f64eaf2b0b99217b65ad79f96979f12adbd9ca49d",
        "train_cache_sha256": "dccfcab2728e223fc1620a00afba24feafe6d47373ec7619632d3e5b80b89789",
        "validation_cache_sha256": "a9fe379a68775dc5808338d69d3061c0dfbf2c1e7ae64b8a1f1f55d57e4e6075",
        "graph_sha256": "a2f35369a5ed4981b8ef75c92df285f7df576a020582f119a1372bfbedbaebee"
      },
      "complete": true,
      "validation": [
        {
          "step": 0,
          "presented_tokens": 0,
          "bits_per_byte": 3.3872027566632044,
          "token_perplexity": 4352.676349513797,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "69570135596f4c2be5faf948d80082e0f792bde7540af2e3af62219a107dacd1"
        },
        {
          "step": 500,
          "presented_tokens": 768000,
          "bits_per_byte": 2.2818513974199326,
          "token_perplexity": 282.6930403551695,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "361f071599200826f41eb49da3f33369dcbf897dc118c79c030e2f5fa9b01eeb"
        },
        {
          "step": 1000,
          "presented_tokens": 1536000,
          "bits_per_byte": 2.103357072726708,
          "token_perplexity": 181.78780783397352,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "ab9b6a18710482adf3cd826a249d076b012b0fc342c44e404acfb3e2a2bf6fba"
        },
        {
          "step": 1500,
          "presented_tokens": 2304000,
          "bits_per_byte": 2.0273546825455226,
          "token_perplexity": 150.63225492152196,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "a9b4445f4c146cb2c891b46bbc74099254d7f662e019e826a93c145f16fc4515"
        },
        {
          "step": 2000,
          "presented_tokens": 3072000,
          "bits_per_byte": 1.970345134202001,
          "token_perplexity": 130.82012638185583,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "1cf190a773ee10e7215f8317214984d5ca7a9a33dab13032e1e0aaea3d7c70ce"
        },
        {
          "step": 2500,
          "presented_tokens": 3840000,
          "bits_per_byte": 1.930792263039493,
          "token_perplexity": 118.62719573796544,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "6431dccba8a2c3295dd1827ab821113480ea07c32bac3907826f8f8a0dea932f"
        },
        {
          "step": 3000,
          "presented_tokens": 4608000,
          "bits_per_byte": 1.9011540944634404,
          "token_perplexity": 110.24146873043483,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "f582b8f82cf7a70084883cf5ec959ec3840fbe0b3b036da67709553d35dd4653"
        },
        {
          "step": 3500,
          "presented_tokens": 5376000,
          "bits_per_byte": 1.871663775908915,
          "token_perplexity": 102.48600102329492,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "2095b42152e69baff7f52ea4671631bebd565c66f4c1955b45da7408a15d5986"
        },
        {
          "step": 4000,
          "presented_tokens": 6144000,
          "bits_per_byte": 1.8536242293413117,
          "token_perplexity": 98.01336008678528,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "92db6abee6755acebd35f609e1b76a3dbce9ccc59ef6fc33b370f5110eecb538"
        },
        {
          "step": 4500,
          "presented_tokens": 6912000,
          "bits_per_byte": 1.8374163471718719,
          "token_perplexity": 94.16157257718444,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "e3291b7f7f86b95e4321b9c349ba5a0640f6183c682c25a14c657c151d546856"
        },
        {
          "step": 5000,
          "presented_tokens": 7680000,
          "bits_per_byte": 1.8247426725359845,
          "token_perplexity": 91.25544907726542,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "cf0bf574511663cd1f486dc485ac67c6ce5f3e17b01eeec2042fe7452a686c8e"
        },
        {
          "step": 5500,
          "presented_tokens": 8448000,
          "bits_per_byte": 1.8180732720673622,
          "token_perplexity": 89.76232586690088,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "b4cda0ee2166cf70a543e81e022b6c7c79a2ec49b732ce5f91971bfbb6260345"
        },
        {
          "step": 6000,
          "presented_tokens": 9216000,
          "bits_per_byte": 1.8138328084572959,
          "token_perplexity": 88.82571496175889,
          "evaluated_tokens": 32768,
          "evaluated_bytes": 116937,
          "artifact_sha256": "40b9ed24172dde0d51b128355df4882d42d452e9f078f76608b8e97b50ed4d7e"
        }
      ],
      "source_commit": "8cf2ab2868ef39f9ea81dc341b028badca66b366"
    }
  ]
}