1e3:["$","div",null,{"children":[["$","p",null,{"children":["Loss falls as a ",["$","strong",null,{"children":"power law"}]," in parameters ",["$","span",null,{"className":"","dangerouslySetInnerHTML":{"__html":"

N

"}}],", data"," ",["$","span",null,{"className":"","dangerouslySetInnerHTML":{"__html":"

D

"}}],", and compute ",["$","span",null,{"className":"","dangerouslySetInnerHTML":{"__html":"

C

"}}],". ",["$","strong",null,{"children":"Kaplan (2020)"}]," said spend extra compute on ",["$","strong",null,{"children":"bigger models"}]," (→ large, under-trained models like GPT-3). ",["$","strong",null,{"children":"Chinchilla (2022)"}]," found ",["$","span",null,{"className":"","dangerouslySetInnerHTML":{"__html":"

N

"}}]," and ",["$","span",null,{"className":"","dangerouslySetInnerHTML":{"__html":"

D

"}}]," ","should scale ",["$","strong",null,{"children":"together"}]," — compute-optimal is ",["$","strong",null,{"children":"~20 tokens per parameter"}],"."]}],["$","h4",null,{"children":"Worked example — the Chinchilla token budget"}],["$","pre",null,{"children":["$","code",null,{"children":"Chinchilla rule: ~20 training tokens per parameter\n\n70B params -> 70e9 * 20 = 1.4T tokens (compute-optimal)\nGPT-3 175B -> trained on ~300B tokens = ~1.7 tokens/param (under-trained)"}]}],["$","p",null,{"children":["A 70B Chinchilla-optimal model wants ",["$","strong",null,{"children":"1.4T"}]," tokens; GPT-3 at 175B saw only ~300B (~1.7 tokens/param — badly under-trained by this rule). Deployment twist: if you'll serve billions of inference tokens, ","$L1e8"," ","past Chinchilla — costlier to train, far cheaper to serve (the LLaMA strategy)."]}]]}]

This module is not available