From 6d9cb74a42a9801d7d8300fe5a9f8ac192c3c1e8 Mon Sep 17 00:00:00 2001 From: Bhimraj Yadav Date: Thu, 24 Sep 2026 02:46:18 +0000 Subject: [PATCH 01/10] Keep pyproject.toml in the sdist The project metadata lives in pyproject.toml's [project] table, but `exclude *.toml` left it out of the sdist. A wheel built from that sdist comes out named `thunder`, without its summary, license or requires-python, and uv rejects it outright. pyproject.toml is the only toml at the root, so drop the exclude and let setuptools include it as it does by default. --- MANIFEST.in | 1 - 1 file changed, 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 4de1c9a3a6..2545a3c6ae 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,7 +8,6 @@ recursive-include pl_sandbox *.md include LICENSE # Exclude build configs -exclude *.toml exclude *.svg exclude *.yml exclude *.yaml From c03303bf8d889353cdcdab8eba479ea0a863ec5a Mon Sep 17 00:00:00 2001 From: Bhimraj Yadav Date: Thu, 24 Sep 2026 02:51:21 +0000 Subject: [PATCH 02/10] Drop unused torchaudio from devel requirements torchaudio is not imported anywhere in the repo. CI pins it in lockstep with torch, but torchaudio stopped tracking torch after 2.11, so the pin no longer resolves and every job fails at dependency install. --- requirements/devel.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/devel.txt b/requirements/devel.txt index 9ffb9b0750..d656b07514 100644 --- a/requirements/devel.txt +++ b/requirements/devel.txt @@ -6,4 +6,3 @@ datasets>=3.5.0 peft>=0.15.2 torchvision -torchaudio From e8f622de78afc318b30a25b98d2b994b52cf8fd0 Mon Sep 17 00:00:00 2001 From: Bhimraj Yadav Date: Thu, 24 Sep 2026 07:50:10 +0000 Subject: [PATCH 03/10] Ignore upstream FutureWarnings that fail test collection pyproject.toml turns every FutureWarning into an error. Three come from code outside thunder and fail tests that have nothing wrong with them: - torch's test utils read the deprecated torch._dynamo.config.inline_inbuilt_nn_modules as a default argument, so importing them fails collection of every distributed test file. - torch.jit.script is deprecated, and torch.func.jvp, which the grads tests use as their reference, imports decompositions_for_jvp, which scripts its decompositions on import. The prefix covers both wordings torch emits. - bitsandbytes still calls torch._check_is_size, which torch deprecates. Ignore exactly those messages. Every other FutureWarning stays fatal. --- pyproject.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 5d40822c5c..30ac5aa106 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,6 +74,13 @@ markers = [ ] filterwarnings = [ "error::FutureWarning", + # todo: drop these once upstream stops raising them + # torch/testing/_internal/common_utils.py reads this config at import time + "ignore:torch._dynamo.config.inline_inbuilt_nn_modules is deprecated:FutureWarning", + # torch.func.jvp scripts its decompositions on import; the prefix also covers the 3.14+ wording + "ignore:`torch.jit.script` is:FutureWarning", + # called by bitsandbytes + "ignore:_check_is_size will be removed:FutureWarning", ] timeout = 900 # xfail_strict = true # todo From bd94969a86a10a6143d3e845ed62598a46bb7405 Mon Sep 17 00:00:00 2001 From: Bhimraj Yadav Date: Thu, 24 Sep 2026 07:51:07 +0000 Subject: [PATCH 04/10] Fix import thunder on PyTorch 2.13 (named tensor removal) torch 2.13 removed named tensors, so Tensor.align_as, align_to, has_names, refine_names and rename are gone. default_torch_ops.py lists them in a module-level dict, so `import thunder` fails with an AttributeError. Only register them on torch versions that still have them. --- thunder/constants.py | 7 ++++++ thunder/tests/test_auto_register_torchops.py | 11 ++++++++++ thunder/torch/default_torch_ops.py | 23 +++++++++++++++----- 3 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 thunder/constants.py diff --git a/thunder/constants.py b/thunder/constants.py new file mode 100644 index 0000000000..5a49f674a0 --- /dev/null +++ b/thunder/constants.py @@ -0,0 +1,7 @@ +import operator + +from lightning_utilities.core.imports import compare_version + +# PyTorch version constants +# `use_base_version` so that dev builds like "2.13.0a0+gitabc123" compare as "2.13.0" +_TORCH_GREATER_EQUAL_2_13 = compare_version("torch", operator.ge, "2.13.0", use_base_version=True) diff --git a/thunder/tests/test_auto_register_torchops.py b/thunder/tests/test_auto_register_torchops.py index 60cf5a3d8a..3bef7f74b8 100644 --- a/thunder/tests/test_auto_register_torchops.py +++ b/thunder/tests/test_auto_register_torchops.py @@ -5,6 +5,7 @@ import pytest import thunder import thunder.torch.default_torch_ops as ops +from thunder.constants import _TORCH_GREATER_EQUAL_2_13 from thunder.torch import _get_torch_function_name import torch @@ -248,3 +249,13 @@ def fn_none(a): cfn(a) ops = thunder.get_auto_registered_torch_op_names(cfn) assert expect == ops + + +def test_named_tensor_ops_follow_torch(): + # torch 2.13 removed named tensors, and this table is built at import time. + named_tensor_methods = {"align_as", "align_to", "has_names", "refine_names", "rename"} + registered = {fn.__name__ for fn in ops.torch_auto_registered_ops[torch.Tensor]} + if _TORCH_GREATER_EQUAL_2_13: + assert not (named_tensor_methods & registered) + else: + assert named_tensor_methods <= registered diff --git a/thunder/torch/default_torch_ops.py b/thunder/torch/default_torch_ops.py index 195625aa3a..15a4879e5a 100644 --- a/thunder/torch/default_torch_ops.py +++ b/thunder/torch/default_torch_ops.py @@ -1,5 +1,21 @@ import torch +from thunder.constants import _TORCH_GREATER_EQUAL_2_13 + +# torch 2.13 removed named tensors (pytorch/pytorch#173895), and this table is built at import +# time, so referencing the removed methods would break `import thunder` there. +_named_tensor_methods = ( + [] + if _TORCH_GREATER_EQUAL_2_13 + else [ + torch.Tensor.align_as, + torch.Tensor.align_to, + torch.Tensor.has_names, + torch.Tensor.refine_names, + torch.Tensor.rename, + ] +) + torch_auto_registered_ops = { torch: [ torch._native_multi_head_attention, @@ -358,8 +374,8 @@ torch.Tensor.addmv, torch.Tensor.addr, torch.Tensor.adjoint, - torch.Tensor.align_as, - torch.Tensor.align_to, + # align_as, align_to, has_names, refine_names and rename, on torch versions that still have them + *_named_tensor_methods, torch.Tensor.aminmax, torch.Tensor.angle, torch.Tensor.arccos, @@ -432,7 +448,6 @@ torch.Tensor.greater, torch.Tensor.greater_equal, torch.Tensor.half, - torch.Tensor.has_names, torch.Tensor.heaviside, torch.Tensor.histc, torch.Tensor.histogram, @@ -520,8 +535,6 @@ torch.Tensor.quantile, torch.Tensor.rad2deg, torch.Tensor.ravel, - torch.Tensor.refine_names, - torch.Tensor.rename, torch.Tensor.renorm, torch.Tensor.reshape_as, torch.Tensor.resize, From ff59da0e47d3c866a1c13664dbd2f4701f5df877 Mon Sep 17 00:00:00 2001 From: Bhimraj Yadav Date: Thu, 24 Sep 2026 08:05:52 +0000 Subject: [PATCH 05/10] Fix the torch.autograd.Function.apply lookaside on PyTorch 2.13 torch 2.13 reimplemented Function.apply in C, so Function.apply.__func__ no longer exists and `import thunder` fails at module scope. Swapping in the bound Function.apply would not work: it is a builtin_function_or_method, and MyFunc.apply is a fresh object that is neither equal nor hash-equal to it, so the identity-keyed lookaside map would never match a user's subclass and every autograd.Function would silently stop being traced. The classmethod_descriptor on Function is the one object all subclasses share, so key the lookaside on that, and have the interpreter resolve builtin classmethods to their descriptor. --- thunder/core/interpreter.py | 10 ++++++++++ thunder/core/jit_ext.py | 11 ++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/thunder/core/interpreter.py b/thunder/core/interpreter.py index 6bc8270940..1d17de8fad 100644 --- a/thunder/core/interpreter.py +++ b/thunder/core/interpreter.py @@ -33,6 +33,7 @@ NoneType, BuiltinFunctionType, BuiltinMethodType, + ClassMethodDescriptorType, MethodDescriptorType, MethodWrapperType, WrapperDescriptorType, @@ -7027,6 +7028,15 @@ def _impl(fn, *args, **kwargs): assert is_opaque(fn) slf = fn.__self__ + # NOTE Builtin classmethods (e.g. torch.autograd.Function.apply since torch 2.13) dispatch + # through their descriptor, the one object all subclasses share, so lookasides can match it. + if isinstance(slf, type) and not is_pycapsule(slf): + for klass in slf.__mro__: + descriptor = klass.__dict__.get(fn.__name__) + if isinstance(descriptor, ClassMethodDescriptorType): + wrapped_slf = _interpret_call(getattr, wrapped_fn, wrap_const("__self__")) + return _interpret_call(wrap_const(descriptor), wrapped_slf, *args, **kwargs) + if slf is not None and not isinstance(slf, (type, ModuleType)) and not is_pycapsule(slf): # NOTE: we need to walk the mro because we need to deal with super().foo # using the qualname is not good here because Python qualnames come with no diff --git a/thunder/core/jit_ext.py b/thunder/core/jit_ext.py index 399ddc471a..95b38e3d94 100644 --- a/thunder/core/jit_ext.py +++ b/thunder/core/jit_ext.py @@ -794,7 +794,16 @@ def _convert_pytorchfunc_to_thundertrace( return trace, sequencify(wrapped_func_result)[0].provenance -@register_general_jit_lookaside(torch.autograd.function.Function.apply.__func__) +# The key must be the object every subclass shares: __func__ for a Python classmethod, +# the descriptor since torch 2.13 reimplemented Function.apply in C. +_torch_autograd_function_apply = getattr( + torch.autograd.function.Function.apply, + "__func__", + torch.autograd.function.Function.__dict__.get("apply"), +) + + +@register_general_jit_lookaside(_torch_autograd_function_apply) def _general_jit_torch_autograd_function_apply_lookaside(obj: Any, *args, **kwargs): """Encapsulate forward into a bsym, define and register augmented fwd and bwd. From a5cca3ed950d522bdb7af9dfa1369320cc19b12a Mon Sep 17 00:00:00 2001 From: Bhimraj Yadav Date: Thu, 24 Sep 2026 11:01:24 +0000 Subject: [PATCH 06/10] Generate a dynamo partition's code before thunder.jit traces it Since torch 2.12, the partitions the splitter hands to thunder.jit are _LazyGraphModules whose code has not been generated yet (on 2.11 they are plain GraphModules). Their first call, and with it GraphModule.recompile(), then runs inside thunder's interpreter, and tracing torch fx codegen fails on what the interpreter doesn't support, which broke every DynamoThunder test: TypeError: object of type 'MappingKeysView' has no len() TypeError: object of type 'generator' has no len() Call recompile_graph first, as the splitter already does for split_gm. It runs real_recompile() on lazy modules and is a plain recompile otherwise. Do the same for the partition's copy in original_split_gm, which ThunderCompilerGraphBenchmarking jits. --- thunder/dynamo/splitter.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/thunder/dynamo/splitter.py b/thunder/dynamo/splitter.py index 11ba9eca0e..7610bf6cea 100644 --- a/thunder/dynamo/splitter.py +++ b/thunder/dynamo/splitter.py @@ -230,14 +230,17 @@ def is_thunder_supported_partition(node: torch.fx.Node) -> bool: example_input_metadatas.append(list(example_input_metadata)) # Replace PyTorch operators within the checkpointed function with the corresponding Thunder operators checkpoint_converter(split_gm, graph_module) + # Since torch 2.12 this is a lazy GraphModule: generate its code now, so thunder doesn't trace fx codegen + recompile_graph(graph_module) jit_fn = thunder_jit(graph_module, is_differentiable_outputs=is_differentiable_outputs) # Update the node name from "submod_*" to "thunder_*" for more user-friendly names update_node_and_submodule(split_gm, node, node.name.replace("submod", "thunder"), jit_fn) thunder_compiled_fns.append(jit_fn) - submodule_to_compiled_fns[getattr(original_split_gm, node_name)] = CompiledFunction( - jit_fn, CompilerType.THUNDER - ) + original_graph_module = getattr(original_split_gm, node_name) + # ThunderCompilerGraphBenchmarking jits this copy too + recompile_graph(original_graph_module) + submodule_to_compiled_fns[original_graph_module] = CompiledFunction(jit_fn, CompilerType.THUNDER) elif node.name.startswith("submod"): # For inductor graph_module = getattr(split_gm, node.name) From 74d1cb5433ac54645b33c57e2f4aedf9939f7593 Mon Sep 17 00:00:00 2001 From: Bhimraj Yadav Date: Thu, 24 Sep 2026 08:30:40 +0000 Subject: [PATCH 07/10] Skip degenerate group_norm samples on torch 2.14+ torch 2.14 rejects group_norm inputs with an empty channel or spatial extent ("Expected number of channels to be greater than 0", "Expected HxW to be greater than 0"); 2.13 and earlier accepted them. The sample generator builds shapes from (0, 2) in every dim, so the torch reference failed on inputs it now refuses. An empty batch is still accepted, so only channels and the spatial dims are filtered. The boundary is 2.14, not 2.13, hence the new constant. --- thunder/constants.py | 1 + thunder/tests/opinfos.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/thunder/constants.py b/thunder/constants.py index 5a49f674a0..29d6e8c292 100644 --- a/thunder/constants.py +++ b/thunder/constants.py @@ -5,3 +5,4 @@ # PyTorch version constants # `use_base_version` so that dev builds like "2.13.0a0+gitabc123" compare as "2.13.0" _TORCH_GREATER_EQUAL_2_13 = compare_version("torch", operator.ge, "2.13.0", use_base_version=True) +_TORCH_GREATER_EQUAL_2_14 = compare_version("torch", operator.ge, "2.14.0", use_base_version=True) diff --git a/thunder/tests/opinfos.py b/thunder/tests/opinfos.py index f6997926de..a39cc5fa2c 100644 --- a/thunder/tests/opinfos.py +++ b/thunder/tests/opinfos.py @@ -23,6 +23,7 @@ import thunder.core.prims as prims from thunder.core.pytree import tree_map from thunder.core.symbol import Symbol +from thunder.constants import _TORCH_GREATER_EQUAL_2_14 import thunder.executors as executors from thunder.tests.framework import _all_devicetypes, custom_comparator, IS_WINDOWS from thunder.tests.make_tensor import make_tensor, make_tensor_like @@ -8574,6 +8575,10 @@ def group_norm_sample_generator(op, device, dtype, requires_grad, **kwargs): if torch.device(device).type == "cuda" and ndim >= 3 and num_channels == 0: continue + # torch 2.14 rejects empty channel or spatial dims (an empty batch is still fine) + if _TORCH_GREATER_EQUAL_2_14 and (num_channels == 0 or math.prod(inner_dims) == 0): + continue + a = make(shape) for weight, bias in itertools.product((False, True), repeat=2): From ae705a29322218fa7f05a596d3835c403f74cf33 Mon Sep 17 00:00:00 2001 From: bhimrazy Date: Tue, 15 Sep 2026 22:54:35 +0545 Subject: [PATCH 08/10] Accept saved_for_backward_idx in autograd_function_apply torch 2.11 replaced args_tensor_mask on the autograd_function_apply higher-order op with a required saved_for_backward_idx (and 2.13 added an optional dirty_idx), so test_autograd_function_apply failed with KeyError: 'saved_for_backward_idx'. The test helper now passes it, and thunder's autograd_function_apply symbol accepts both new kwargs, which test_autograd_function_apply_with_no_grad passes through. thunder does not use either, same as non_differentiable_idx. --- thunder/tests/test_jit_general.py | 3 +++ thunder/torch/__init__.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/thunder/tests/test_jit_general.py b/thunder/tests/test_jit_general.py index 15a62703ee..a0fb7671bc 100644 --- a/thunder/tests/test_jit_general.py +++ b/thunder/tests/test_jit_general.py @@ -47,6 +47,9 @@ def _autograd_function_apply_kwargs(args_tensor_mask, non_differentiable_idx=Non kwargs = {} if _HAS_ARGS_TENSOR_MASK: kwargs["args_tensor_mask"] = args_tensor_mask + else: + # required since torch 2.11; the tests here save only x + kwargs["saved_for_backward_idx"] = [0] if non_differentiable_idx is not None: kwargs["non_differentiable_idx"] = non_differentiable_idx return kwargs diff --git a/thunder/torch/__init__.py b/thunder/torch/__init__.py index 74ae2ba269..417043b2b0 100644 --- a/thunder/torch/__init__.py +++ b/thunder/torch/__init__.py @@ -6715,6 +6715,8 @@ def autograd_function_apply( *args: Any, args_tensor_mask: Sequence[bool] | None = None, non_differentiable_idx: Sequence[int] | None = None, + saved_for_backward_idx: Sequence[int] | None = None, # unused, passed by torch >= 2.11 + dirty_idx: Sequence[int] | None = None, # unused, passed by torch >= 2.13 ) -> TensorProxy | tuple[TensorProxy, ...]: # TODO: Remove this once this autograd API becomes stable. # On stable PyTorch, fwd expects ctx as first argument @@ -6733,6 +6735,8 @@ def augmented_forward_autograd_function_apply( *args: Any, args_tensor_mask: Sequence[bool] | None = None, non_differentiable_idx: Sequence[int] | None = None, + saved_for_backward_idx: Sequence[int] | None = None, # unused, passed by torch >= 2.11 + dirty_idx: Sequence[int] | None = None, # unused, passed by torch >= 2.13 ) -> tuple[TensorProxy | tuple[TensorProxy, ...], tuple[Any, ...]]: # TODO: Remove this once this autograd API becomes stable. # On stable PyTorch, fwd expects ctx as first argument From f264a8a1cd9f6ca0dffdfc5fd4e5dd1dde8c5c8c Mon Sep 17 00:00:00 2001 From: Bhimraj Yadav Date: Thu, 24 Sep 2026 11:01:36 +0000 Subject: [PATCH 09/10] Keep the tuple the autograd_function_apply fwd graph returns Since torch 2.12, the fwd subgraph dynamo builds returns its outputs as a tuple, and the caller indexes it: return ((y,), (l_x_,)) getitem = autograd_function_apply[0] python_return(*sequencify(output)) unpacked a one-element tuple to the bare tensor, so [0] indexed into the tensor instead, and thunderfx returned a scalar where eager gives f32[2]. test_higher_order_inplace_alias_update: AssertionError: The values for attribute 'shape' do not match: torch.Size([]) != torch.Size([2]). Return the structure the graph produced. Bare and multi-element outputs are unchanged. --- thunder/core/jit_ext.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/thunder/core/jit_ext.py b/thunder/core/jit_ext.py index 95b38e3d94..18fb6792dc 100644 --- a/thunder/core/jit_ext.py +++ b/thunder/core/jit_ext.py @@ -1009,7 +1009,11 @@ def _generate_random_str_id() -> str: continue trace_of_forward.bound_symbols.append(bsym.from_bsym()) with tracectx(trace_of_forward): - prims.python_return(*(sequencify(output))) + # Keep the fwd graph's output structure. Since torch 2.12 it returns a tuple, which the caller indexes + if isinstance(output, (tuple, list)): + prims.python_return(tuple(output)) + else: + prims.python_return(output) # See NOTE: `autograd_function_apply` and `no_grad` interaction for details about # `thunder.torch.call_higher_order_function_and_consider_outer_autograd_setting` From 96a8739177f91b09ec4f7bd0ffbb52fc8107c144 Mon Sep 17 00:00:00 2001 From: Bhimraj Yadav Date: Thu, 24 Sep 2026 02:55:40 +0000 Subject: [PATCH 10/10] Skip the bfloat16 GPT tests on Windows Running these models in bfloat16 on CPU intermittently kills the Windows test process with an illegal instruction (0xc000001d). It happens in eager torch, before thunder runs, so pytest cannot catch it and the whole xdist worker dies. The fp32 variants of the same models still run on Windows, and these still run on Linux and macOS. --- thunder/tests/framework.py | 6 ++++++ thunder/tests/test_interpreter.py | 4 +++- thunder/tests/test_jit_general.py | 4 +++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/thunder/tests/framework.py b/thunder/tests/framework.py index 7fa3b1485b..5265ddfe1a 100644 --- a/thunder/tests/framework.py +++ b/thunder/tests/framework.py @@ -58,6 +58,12 @@ class NOTHING: ) IS_WINDOWS = platform.system() == "Windows" +# NOTE Running these models in bfloat16 on CPU intermittently kills the process on Windows with an +# illegal instruction, inside eager torch rather than anything of ours. The fp32 variants of the +# same models still run there, and these still run on Linux and macOS. +# TODO: Remove this skip once the crash is fixed upstream, or pinned to a oneDNN/MKL code path we can avoid. +WINDOWS_GPT_CRASH_REASON = "bfloat16 CPU model intermittently crashes the process on Windows (0xc000001d)" + def _bitsandbytes_available(): if not package_available("bitsandbytes"): diff --git a/thunder/tests/test_interpreter.py b/thunder/tests/test_interpreter.py index c545954069..fe36f26e64 100644 --- a/thunder/tests/test_interpreter.py +++ b/thunder/tests/test_interpreter.py @@ -11,7 +11,7 @@ import pytest import torch from torch.testing import assert_close -from thunder.tests.framework import IS_WINDOWS +from thunder.tests.framework import IS_WINDOWS, WINDOWS_GPT_CRASH_REASON import thunder from thunder.core.interpreter import ( @@ -3413,6 +3413,7 @@ def test_nanogpt_block(jit): assert_close(result, fn(*args, **kwargs)) +@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_GPT_CRASH_REASON) def test_nanogpt(jit): from thunder.benchmarks import NanoGPTBenchmark, NanoGPTConfig, _nanogpt_configs @@ -3428,6 +3429,7 @@ def test_nanogpt(jit): assert_close(result, fn(*args, **kwargs)) +@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_GPT_CRASH_REASON) def test_litgpt(jit): from thunder.benchmarks import LitGPTBenchmark from thunder.tests.litgpt_model import Config diff --git a/thunder/tests/test_jit_general.py b/thunder/tests/test_jit_general.py index a0fb7671bc..22b42e33d4 100644 --- a/thunder/tests/test_jit_general.py +++ b/thunder/tests/test_jit_general.py @@ -14,7 +14,7 @@ import thunder -from thunder.tests.framework import requiresCUDA, IS_WINDOWS +from thunder.tests.framework import requiresCUDA, IS_WINDOWS, WINDOWS_GPT_CRASH_REASON from thunder.core.options import CACHE_OPTIONS import thunder.core.prims as prims from thunder import pytorch_executor, nvfuser_executor @@ -584,6 +584,7 @@ def h(d, c): assert args_names == ("a", "b", "c", "d") +@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_GPT_CRASH_REASON) def test_litgpt(): from thunder.benchmarks import LitGPTBenchmark from thunder.tests.litgpt_model import Config @@ -664,6 +665,7 @@ def test_nanogpt_mlp(): assert_close(result, module(*args, **kwargs)) +@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_GPT_CRASH_REASON) def test_nanogpt(): from thunder.benchmarks import NanoGPTBenchmark, NanoGPTConfig, _nanogpt_configs