{"TotalCount":60,"Files":[{"Ident":"jeffskitchen.llm_poc","Path":"ui/controls/vectorcontrol.cs.scss","FileName":"vectorcontrol.cs.scss","PackageType":"game","CodeKind":"Game","AssetVersionId":342521,"Code":"VectorControl\r\n{\r\n\tgap: 2px;\r\n\tflex-grow: 1;\r\n}\r\n\r\nVectorControl NumberEntry\r\n{\r\n\tflex-basis: 50%;\r\n}"},{"Ident":"jeffskitchen.llm_poc","Path":"ui/controlsheet/controlsheetgroupheader.cs.scss","FileName":"controlsheetgroupheader.cs.scss","PackageType":"game","CodeKind":"Game","AssetVersionId":342521,"Code":"ControlSheetGroupHeader\r\n{\r\n\tfont-size: 1.33rem;\r\n\tcolor: red;\r\n\tgap: 2px;\r\n\talign-items: center;\r\n\r\n\t\u0026.hidden\r\n\t{\r\n\t\tdisplay: none;\r\n\t}\r\n\r\n\t\u003E .title\r\n\t{\r\n\t\tfont-weight: 600;\r\n\t}\r\n\r\n\t\u0026.has-toggle\r\n\t{\r\n\t\tcursor: pointer;\r\n\t\topacity: 0.8;\r\n\r\n\t\t\u0026:before\r\n\t\t{\r\n\t\t\tcontent: \u0027 \u0027;\r\n\t\t\twidth: 22px;\r\n\t\t\theight: 22px;\r\n\t\t\tbackground-color: #000a;\r\n\t\t\talign-items: center;\r\n\t\t\tjustify-content: center;\r\n\t\t\ttext-align: center;\r\n\t\t\tborder-radius: 5px;\r\n\t\t\tborder: 1px solid #555;\r\n\t\t}\r\n\r\n\t\t\u0026:hover\r\n\t\t{\r\n\t\t\topacity: 1;\r\n\r\n\t\t\t\u0026:before\r\n\t\t\t{\r\n\t\t\t\tborder-color: #888;\r\n\t\t\t}\r\n\t\t}\r\n\r\n\t\t\u0026.checked\r\n\t\t{\r\n\t\t\t\u003E .title\r\n\t\t\t{\r\n\t\t\t\tcolor: white;\r\n\t\t\t}\r\n\r\n\t\t\t\u0026:before\r\n\t\t\t{\r\n\t\t\t\tcontent: \u0027\u2713\u0027;\r\n\t\t\t\tfont-weight: bold;\r\n\t\t\t\tcolor: #08f;\r\n\t\t\t\tborder-color: #08f;\r\n\t\t\t}\r\n\t\t}\r\n\t}\r\n}\r\n"},{"Ident":"jeffskitchen.llm_poc","Path":"styles/form/_checkbox.scss","FileName":"_checkbox.scss","PackageType":"game","CodeKind":"Game","AssetVersionId":342521,"Code":"\r\n@import \u0022/styles/_theme.scss\u0022;\r\n\r\n$primary: $primary-blue !default;\r\n$primary-alt: white !default;\r\n$form-control-height: 24px !default;\r\n\r\n.checkbox\r\n{\r\n\tcursor: pointer;\r\n\tcolor: rgba( $primary-alt, 0.6 );\r\n\talign-items: center;\r\n\tgap: 8px;\r\n\r\n\tlabel\r\n\t{\r\n\t\tpointer-events: none;\r\n\t}\r\n\r\n\t\u003E .checkmark\r\n\t{\r\n\t\tpadding: 1px;\r\n\t\tfont-size: 22px;\r\n\t\tborder: 1px solid $primary;\r\n\t\tborder-radius: $rounding-small;\r\n\t\ttext-align: center;\r\n\t\tjustify-content: center;\r\n\t\talign-items: center;\r\n\t\tcolor: transparent;\r\n\t\tmin-height: $form-control-height;\r\n\t\tpointer-events: none;\r\n\t\tflex-shrink: 0;\r\n\t}\r\n\r\n\t\u0026.checked\r\n\t{\r\n\t\t\u003E .checkmark\r\n\t\t{\r\n\t\t\tcolor: $primary-alt;\r\n\t\t\tbackground-color: $primary;\r\n\t\t}\r\n\t}\r\n\r\n\t\u0026:active\r\n\t{\r\n\t\tcolor: $primary-alt;\r\n\t}\r\n\r\n\t\u0026:hover\r\n\t{\r\n\t\tcolor: $primary-alt;\r\n\t}\r\n}\r\n"},{"Ident":"jeffskitchen.llm_poc","Path":"Llm/GreedyGenerationReference.cs","FileName":"GreedyGenerationReference.cs","PackageType":"game","CodeKind":"Game","AssetVersionId":342521,"Code":"using System.Text.Json.Serialization;\n\nnamespace LlmPoc.Llm;\n\npublic sealed class GreedyGenerationReferenceDocument\n{\n\t[JsonPropertyName( \u0022model_id\u0022 )]\n\tpublic string ModelId { get; set; }\n\n\t[JsonPropertyName( \u0022prompt\u0022 )]\n\tpublic string Prompt { get; set; }\n\n\t[JsonPropertyName( \u0022input_token_ids\u0022 )]\n\tpublic int[] InputTokenIds { get; set; }\n\n\t[JsonPropertyName( \u0022bos_token_id\u0022 )]\n\tpublic int BosTokenId { get; set; }\n\n\t[JsonPropertyName( \u0022eos_token_id\u0022 )]\n\tpublic int EosTokenId { get; set; }\n\n\t[JsonPropertyName( \u0022bos_automatically_added\u0022 )]\n\tpublic bool BosAutomaticallyAdded { get; set; }\n\n\t[JsonPropertyName( \u0022eos_automatically_added\u0022 )]\n\tpublic bool EosAutomaticallyAdded { get; set; }\n\n\t[JsonPropertyName( \u0022do_sample\u0022 )]\n\tpublic bool DoSample { get; set; }\n\n\t[JsonPropertyName( \u0022argmax_tie_break\u0022 )]\n\tpublic string ArgmaxTieBreak { get; set; }\n\n\t[JsonPropertyName( \u0022logits_processors\u0022 )]\n\tpublic string[] LogitsProcessors { get; set; }\n\n\t[JsonPropertyName( \u0022raw_logits_used_for_argmax\u0022 )]\n\tpublic bool RawLogitsUsedForArgmax { get; set; }\n\n\t[JsonPropertyName( \u0022cached_and_uncached_sequences_match\u0022 )]\n\tpublic bool CachedAndUncachedSequencesMatch { get; set; }\n\n\t[JsonPropertyName( \u0022csharp_baseline_use_cache\u0022 )]\n\tpublic bool CsharpBaselineUseCache { get; set; }\n\n\t[JsonPropertyName( \u0022max_new_tokens\u0022 )]\n\tpublic int MaxNewTokens { get; set; }\n\n\t[JsonPropertyName( \u0022generated_token_ids\u0022 )]\n\tpublic int[] GeneratedTokenIds { get; set; }\n\n\t[JsonPropertyName( \u0022generated_text\u0022 )]\n\tpublic string GeneratedText { get; set; }\n\n\t[JsonPropertyName( \u0022full_sequence_token_ids\u0022 )]\n\tpublic int[] FullSequenceTokenIds { get; set; }\n\n\t[JsonPropertyName( \u0022stop_reason\u0022 )]\n\tpublic string StopReason { get; set; }\n\n\t[JsonPropertyName( \u0022eos_reached\u0022 )]\n\tpublic bool EosReached { get; set; }\n\n\t[JsonPropertyName( \u0022minimum_top1_top2_margin\u0022 )]\n\tpublic float MinimumTop1Top2Margin { get; set; }\n\n\t[JsonPropertyName( \u0022minimum_margin_step\u0022 )]\n\tpublic int MinimumMarginStep { get; set; }\n\n\t[JsonPropertyName( \u0022steps\u0022 )]\n\tpublic GreedyGenerationReferenceStep[] Steps { get; set; }\n\n\t[JsonPropertyName( \u0022later_full_logit_reference\u0022 )]\n\tpublic GreedyGenerationLogitReference LaterFullLogitReference { get; set; }\n}\n\npublic sealed class GreedyGenerationReferenceStep\n{\n\t[JsonPropertyName( \u0022step\u0022 )]\n\tpublic int Step { get; set; }\n\n\t[JsonPropertyName( \u0022input_sequence_length\u0022 )]\n\tpublic int InputSequenceLength { get; set; }\n\n\t[JsonPropertyName( \u0022input_token_ids\u0022 )]\n\tpublic int[] InputTokenIds { get; set; }\n\n\t[JsonPropertyName( \u0022new_token_position\u0022 )]\n\tpublic int NewTokenPosition { get; set; }\n\n\t[JsonPropertyName( \u0022expected_next_token_id\u0022 )]\n\tpublic int ExpectedNextTokenId { get; set; }\n\n\t[JsonPropertyName( \u0022decoded_token\u0022 )]\n\tpublic string DecodedToken { get; set; }\n\n\t[JsonPropertyName( \u0022top1_logit\u0022 )]\n\tpublic float Top1Logit { get; set; }\n\n\t[JsonPropertyName( \u0022top2_token_id\u0022 )]\n\tpublic int Top2TokenId { get; set; }\n\n\t[JsonPropertyName( \u0022top2_logit\u0022 )]\n\tpublic float Top2Logit { get; set; }\n\n\t[JsonPropertyName( \u0022top1_top2_margin\u0022 )]\n\tpublic float Top1Top2Margin { get; set; }\n\n\t[JsonPropertyName( \u0022eos_reached\u0022 )]\n\tpublic bool EosReached { get; set; }\n\n\t[JsonPropertyName( \u0022top5\u0022 )]\n\tpublic ForwardReferenceTopLogit[] TopFive { get; set; }\n}\n\npublic sealed class GreedyGenerationLogitReference\n{\n\t[JsonPropertyName( \u0022step\u0022 )]\n\tpublic int Step { get; set; }\n\n\t[JsonPropertyName( \u0022input_sequence_length\u0022 )]\n\tpublic int InputSequenceLength { get; set; }\n\n\t[JsonPropertyName( \u0022file\u0022 )]\n\tpublic string File { get; set; }\n\n\t[JsonPropertyName( \u0022elements\u0022 )]\n\tpublic int Elements { get; set; }\n\n\t[JsonPropertyName( \u0022bytes\u0022 )]\n\tpublic int Bytes { get; set; }\n\n\t[JsonPropertyName( \u0022sha256\u0022 )]\n\tpublic string Sha256 { get; set; }\n}\n"},{"Ident":"jeffskitchen.llm_poc","Path":"Llm/LlmPaths.cs","FileName":"LlmPaths.cs","PackageType":"game","CodeKind":"Game","AssetVersionId":342521,"Code":"namespace LlmPoc.Llm;\n\npublic static class LlmPaths\n{\n\tpublic const string RuntimeModelResource = \u0022models/tinystories.llmmdl\u0022;\n\tpublic const string Root = \u0022models/tinystories-instruct-1m\u0022;\n\tpublic const string Model = Root \u002B \u0022/model.bin\u0022;\n\tpublic const string Config = Root \u002B \u0022/config.json\u0022;\n\tpublic const string Manifest = Root \u002B \u0022/tensor_manifest.json\u0022;\n\tpublic const string Reference = Root \u002B \u0022/reference.json\u0022;\n\tpublic const string ReferenceLogits = Root \u002B \u0022/reference_logits.f32\u0022;\n\tpublic const string ReferenceIntermediates = Root \u002B \u0022/reference_intermediates.json\u0022;\n\tpublic const string ReferenceEmbedding = Root \u002B \u0022/reference_embedding.f32\u0022;\n\tpublic const string ReferenceLayer0Ln1 = Root \u002B \u0022/reference_layer0_ln1.f32\u0022;\n\tpublic const string ReferenceLayer0Q = Root \u002B \u0022/reference_layer0_q.f32\u0022;\n\tpublic const string ReferenceLayer0K = Root \u002B \u0022/reference_layer0_k.f32\u0022;\n\tpublic const string ReferenceLayer0V = Root \u002B \u0022/reference_layer0_v.f32\u0022;\n\tpublic const string ReferenceLayer0QHeads = Root \u002B \u0022/reference_layer0_q_heads.f32\u0022;\n\tpublic const string ReferenceLayer0KHeads = Root \u002B \u0022/reference_layer0_k_heads.f32\u0022;\n\tpublic const string ReferenceLayer0VHeads = Root \u002B \u0022/reference_layer0_v_heads.f32\u0022;\n\tpublic const string ReferenceLayer0ScoresScaledUnmasked =\n\t\tRoot \u002B \u0022/reference_layer0_scores_scaled_unmasked.f32\u0022;\n\tpublic const string ReferenceLayer0AttentionMask =\n\t\tRoot \u002B \u0022/reference_layer0_attention_mask.json\u0022;\n\tpublic const string ReferenceLayer0ScoresMaskedPreSoftmax =\n\t\tRoot \u002B \u0022/reference_layer0_scores_masked_pre_softmax.f32\u0022;\n\tpublic const string ReferenceLayer0AttentionProbs =\n\t\tRoot \u002B \u0022/reference_layer0_attention_probs.f32\u0022;\n\tpublic const string ReferenceLayer0AttentionContextHeads =\n\t\tRoot \u002B \u0022/reference_layer0_attention_context_heads.f32\u0022;\n\tpublic const string ReferenceLayer0AttentionMerged =\n\t\tRoot \u002B \u0022/reference_layer0_attention_merged.f32\u0022;\n\tpublic const string ReferenceLayer0AttentionOutProj =\n\t\tRoot \u002B \u0022/reference_layer0_attention_out_proj.f32\u0022;\n\tpublic const string ReferenceLayer0AttentionResidual =\n\t\tRoot \u002B \u0022/reference_layer0_attention_residual.f32\u0022;\n\tpublic const string ReferenceLayer0Ln2 = Root \u002B \u0022/reference_layer0_ln2.f32\u0022;\n\tpublic const string ReferenceLayer0MlpFc = Root \u002B \u0022/reference_layer0_mlp_fc.f32\u0022;\n\tpublic const string ReferenceLayer0MlpGelu = Root \u002B \u0022/reference_layer0_mlp_gelu.f32\u0022;\n\tpublic const string ReferenceLayer0MlpProj = Root \u002B \u0022/reference_layer0_mlp_proj.f32\u0022;\n\tpublic const string ReferenceLayer0Output = Root \u002B \u0022/reference_layer0_output.f32\u0022;\n\tpublic const string ReferenceFinalLayerNorm = Root \u002B \u0022/reference_final_ln.f32\u0022;\n\tpublic const string ReferenceFinalLogitsLastPosition =\n\t\tRoot \u002B \u0022/reference_final_logits_last_position.f32\u0022;\n\tpublic const string ReferenceGreedyGeneration =\n\t\tRoot \u002B \u0022/reference_greedy_generation.json\u0022;\n\tpublic const string ReferenceGenerationStep11Logits =\n\t\tRoot \u002B \u0022/reference_generation_step11_logits.f32\u0022;\n\tpublic const string ReferenceLayer1LocalMaskLen260 =\n\t\tRoot \u002B \u0022/reference_layer1_local_mask_len260.bin\u0022;\n\tpublic const string ReferenceLayer1LocalMaskLen260Metadata =\n\t\tRoot \u002B \u0022/reference_layer1_local_mask_len260.json\u0022;\n\tpublic const string Tokenizer = Root \u002B \u0022/tokenizer/tokenizer.json\u0022;\n\tpublic const string TokenizerConfig = Root \u002B \u0022/tokenizer/tokenizer_config.json\u0022;\n\n\tpublic static string ReferenceStage( string fileName )\n\t{\n\t\tif ( string.IsNullOrWhiteSpace( fileName ) || fileName.Contains( \u0027/\u0027 ) ||\n\t\t\tfileName.Contains( \u0027\\\\\u0027 ) || fileName.Contains( \u0022..\u0022 ) )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t$\u0022[LLM:ERROR] Reference stage filename \u0027{fileName}\u0027 is invalid.\u0022,\n\t\t\t\tnameof( fileName ) );\n\t\t}\n\t\treturn Root \u002B \u0022/\u0022 \u002B fileName;\n\t}\n}\n"},{"Ident":"jeffskitchen.llm_poc","Path":".obj/__compiler_extra.cs","FileName":"__compiler_extra.cs","PackageType":"game","CodeKind":"Game","AssetVersionId":342521,"Code":"global using static Sandbox.Internal.GlobalGameNamespace;\r\nglobal using Microsoft.AspNetCore.Components;\r\nglobal using Microsoft.AspNetCore.Components.Rendering;\r\n[assembly: global::System.Reflection.AssemblyMetadata( \u0022AddonTitle\u0022, \u0022LLM POC\u0022 )]\r\n[assembly: global::System.Reflection.AssemblyMetadata( \u0022AddonIdent\u0022, \u0022llm_poc\u0022 )]\r\n[assembly: global::System.Reflection.AssemblyMetadata( \u0022OrgIdent\u0022, \u0022jeffskitchen\u0022 )]\r\n[assembly: global::System.Reflection.AssemblyMetadata( \u0022Ident\u0022, \u0022jeffskitchen.llm_poc\u0022 )]\r\n[assembly: global::System.Reflection.AssemblyMetadata( \u0022EngineVersion\u0022, \u002228\u0022 )]\r\n[assembly: global::System.Reflection.AssemblyMetadata( \u0022EngineMinorVersion\u0022, \u00221\u0022 )]\r\n\r\n[assembly: System.Runtime.Versioning.TargetFramework( \u0022.NETCoreApp,Version=v9.0\u0022, FrameworkDisplayName = \u0022.NET 9.0\u0022 )]\r\n[assembly: global::System.Reflection.AssemblyMetadata( \u0022CompileTime\u0022, \u00222026-08-17T04:57:47.5628694Z\u0022 )]\r\n[assembly: global::System.Reflection.AssemblyVersion(\u00220.0.124.0\u0022)]\r\n[assembly: global::System.Reflection.AssemblyFileVersion(\u00220.0.124.0\u0022)]"},{"Ident":"jeffskitchen.llm_poc","Path":"Assembly.cs","FileName":"Assembly.cs","PackageType":"game","CodeKind":"Game","AssetVersionId":342521,"Code":"global using Sandbox;\nglobal using System;\nglobal using System.Collections.Generic;\nglobal using System.Linq;\nglobal using System.Threading.Tasks;\n"},{"Ident":"jeffskitchen.llm_poc","Path":"Llm/GreedyGenerationParity.cs","FileName":"GreedyGenerationParity.cs","PackageType":"game","CodeKind":"Game","AssetVersionId":342521,"Code":"using Sandbox.Diagnostics;\n\nnamespace LlmPoc.Llm;\n\npublic sealed class GreedyGenerationParityResult\n{\n\tpublic GreedyGenerationResult Generation { get; init; }\n\tpublic NumericComparison LaterStepLogitComparison { get; init; }\n\tpublic int LaterStepLogitIndex { get; init; }\n\tpublic float MinimumMargin { get; init; }\n\tpublic int MinimumMarginStep { get; init; }\n\tpublic double ValidationHarnessMilliseconds { get; init; }\n}\n\npublic static class GreedyGenerationParity\n{\n\tprivate const double LogitAbsoluteTolerance = 5.0e-5;\n\tprivate const double LogitRelativeTolerance = 1.0e-5;\n\tprivate const double TopLogitAbsoluteTolerance = 5.0e-5;\n\n\tpublic static GreedyGenerationParityResult Validate(\n\t\tSboxLlmModel model,\n\t\tTinyStoriesConfig config,\n\t\tGpt2ByteBpeTokenizer tokenizer,\n\t\tLlmReferenceData historicalReference,\n\t\tGreedyGenerationReferenceDocument reference )\n\t{\n\t\tif ( model is null ) throw new ArgumentNullException( nameof( model ) );\n\t\tif ( config is null ) throw new ArgumentNullException( nameof( config ) );\n\t\tif ( tokenizer is null ) throw new ArgumentNullException( nameof( tokenizer ) );\n\t\tif ( historicalReference is null )\n\t\t\tthrow new ArgumentNullException( nameof( historicalReference ) );\n\t\tif ( reference is null ) throw new ArgumentNullException( nameof( reference ) );\n\n\t\tValidateReference( config, tokenizer, historicalReference, reference );\n\t\tfloat[] laterExpected = ReferenceFloatData.LoadFromMounted(\n\t\t\tLlmPaths.ReferenceStage( reference.LaterFullLogitReference.File ),\n\t\t\treference.LaterFullLogitReference.Elements );\n\t\tNumericComparison laterComparison = null;\n\t\tFastTimer validationTimer = FastTimer.StartNew();\n\n\t\tGreedyGenerationResult generation = TinyStoriesGreedyGenerator.Generate(\n\t\t\tmodel,\n\t\t\tconfig,\n\t\t\ttokenizer,\n\t\t\treference.InputTokenIds,\n\t\t\treference.MaxNewTokens,\n\t\t\tobservation =\u003E\n\t\t\t{\n\t\t\t\tGreedyGenerationStepResult actual = observation.Step;\n\t\t\t\tif ( actual.Step \u003C 0 || actual.Step \u003E= reference.Steps.Length )\n\t\t\t\t{\n\t\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t\t$\u0022[LLM:ERROR] Generation produced unexpected step {actual.Step}.\u0022 );\n\t\t\t\t}\n\t\t\t\tGreedyGenerationReferenceStep expected = reference.Steps[actual.Step];\n\t\t\t\tValidateStepContext( expected, observation.InputTokenIds );\n\t\t\t\tbool tokenPassed = actual.TokenId == expected.ExpectedNextTokenId;\n\t\t\t\tif ( !tokenPassed )\n\t\t\t\t{\n\t\t\t\t\tLogMismatch( tokenizer, expected, observation );\n\t\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t\t$\u0022[LLM:ERROR] Greedy generation first mismatch at step {actual.Step}: \u0022 \u002B\n\t\t\t\t\t\t$\u0022expected token {expected.ExpectedNextTokenId}, actual {actual.TokenId}. \u0022 \u002B\n\t\t\t\t\t\t\u0022No mismatching token was appended.\u0022 );\n\t\t\t\t}\n\n\t\t\t\tif ( actual.DecodedToken != expected.DecodedToken )\n\t\t\t\t{\n\t\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t\t$\u0022[LLM:ERROR] Generation step {actual.Step} token {actual.TokenId} decoded \u0022 \u002B\n\t\t\t\t\t\t$\u0022as \u0027{EscapeVisible( actual.DecodedToken )}\u0027, Python expected \u0022 \u002B\n\t\t\t\t\t\t$\u0022\u0027{EscapeVisible( expected.DecodedToken )}\u0027.\u0022 );\n\t\t\t\t}\n\t\t\t\tValidateTopFive( tokenizer, expected, actual );\n\t\t\t\tif ( actual.EosReached != expected.EosReached )\n\t\t\t\t{\n\t\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t\t$\u0022[LLM:ERROR] Generation step {actual.Step} EOS state expected \u0022 \u002B\n\t\t\t\t\t\t$\u0022{expected.EosReached}, actual {actual.EosReached}.\u0022 );\n\t\t\t\t}\n\n\t\t\t\tif ( actual.Step == reference.LaterFullLogitReference.Step )\n\t\t\t\t{\n\t\t\t\t\tlaterComparison = TensorDiagnostics.Compare(\n\t\t\t\t\t\tlaterExpected,\n\t\t\t\t\t\tobservation.Forward.Logits.Data,\n\t\t\t\t\t\tLogitAbsoluteTolerance,\n\t\t\t\t\t\tLogitRelativeTolerance );\n\t\t\t\t\tTensorSummary summary = TensorDiagnostics.Summarize(\n\t\t\t\t\t\tobservation.Forward.Logits );\n\t\t\t\t\tLlmLog.Info(\n\t\t\t\t\t\t\u0022PARITY\u0022,\n\t\t\t\t\t\t$\u0022generation.step{actual.Step}.logits count={summary.ElementCount:N0} \u0022 \u002B\n\t\t\t\t\t\t$\u0022finite={summary.FiniteCount:N0}/{summary.ElementCount:N0} \u0022 \u002B\n\t\t\t\t\t\t$\u0022maxAbs={laterComparison.MaximumAbsoluteError:G12} \u0022 \u002B\n\t\t\t\t\t\t$\u0022meanAbs={laterComparison.MeanAbsoluteError:G12} \u0022 \u002B\n\t\t\t\t\t\t$\u0022maxRel={laterComparison.MaximumRelativeError:G12} \u0022 \u002B\n\t\t\t\t\t\t$\u0022worst_vocab={laterComparison.MaximumErrorIndex} \u0022 \u002B\n\t\t\t\t\t\t$\u0022expected={laterComparison.ExpectedAtMaximumError:G9} \u0022 \u002B\n\t\t\t\t\t\t$\u0022actual={laterComparison.ActualAtMaximumError:G9} \u0022 \u002B\n\t\t\t\t\t\t$\u0022absTol={LogitAbsoluteTolerance:G1} relTol={LogitRelativeTolerance:G1} \u0022 \u002B\n\t\t\t\t\t\t$\u0022{(laterComparison.Passed ? \u0022PASS\u0022 : \u0022FAIL\u0022)}\u0022 );\n\t\t\t\t\tif ( !laterComparison.Passed )\n\t\t\t\t\t{\n\t\t\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t\t\t$\u0022[LLM:ERROR] Later generation step {actual.Step} full-logit parity \u0022 \u002B\n\t\t\t\t\t\t\t$\u0022failed: {laterComparison}.\u0022 );\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tLlmLog.Info(\n\t\t\t\t\t\u0022GEN\u0022,\n\t\t\t\t\t$\u0022step={actual.Step} context={actual.InputSequenceLength} \u0022 \u002B\n\t\t\t\t\t$\u0022position={actual.NewTokenPosition} expected={expected.ExpectedNextTokenId} \u0022 \u002B\n\t\t\t\t\t$\u0022actual={actual.TokenId} token=\u0027{EscapeVisible( actual.DecodedToken )}\u0027 \u0022 \u002B\n\t\t\t\t\t$\u0022margin={actual.Top1Top2Margin:G9} forward_ms={actual.ForwardMilliseconds:N4} \u0022 \u002B\n\t\t\t\t\t$\u0022lm_head_ms={actual.LmHeadMilliseconds:N4} PASS\u0022 );\n\t\t\t} );\n\n\t\tRequireExactArray(\n\t\t\t\u0022generated token IDs\u0022, reference.GeneratedTokenIds, generation.GeneratedTokenIds );\n\t\tRequireExactArray(\n\t\t\t\u0022full generated sequence\u0022, reference.FullSequenceTokenIds,\n\t\t\tgeneration.FullSequenceTokenIds );\n\t\tif ( generation.GeneratedText != reference.GeneratedText )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] Generated text expected \u0027{EscapeVisible( reference.GeneratedText )}\u0027, \u0022 \u002B\n\t\t\t\t$\u0022actual \u0027{EscapeVisible( generation.GeneratedText )}\u0027.\u0022 );\n\t\t}\n\t\tif ( generation.StopReason != reference.StopReason ||\n\t\t\tgeneration.EosReached != reference.EosReached )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] Generation stop expected reason={reference.StopReason} \u0022 \u002B\n\t\t\t\t$\u0022eos={reference.EosReached}, actual reason={generation.StopReason} \u0022 \u002B\n\t\t\t\t$\u0022eos={generation.EosReached}.\u0022 );\n\t\t}\n\t\tif ( laterComparison is null )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] Later-step full-logit checkpoint \u0022 \u002B\n\t\t\t\t$\u0022{reference.LaterFullLogitReference.Step} did not execute.\u0022 );\n\t\t}\n\n\t\tGreedyGenerationStepResult minimum = generation.Steps[0];\n\t\tfor ( int index = 1; index \u003C generation.Steps.Length; index\u002B\u002B )\n\t\t{\n\t\t\tif ( generation.Steps[index].Top1Top2Margin \u003C minimum.Top1Top2Margin )\n\t\t\t{\n\t\t\t\tminimum = generation.Steps[index];\n\t\t\t}\n\t\t}\n\t\tif ( minimum.Step != reference.MinimumMarginStep ||\n\t\t\tMath.Abs( minimum.Top1Top2Margin - reference.MinimumTop1Top2Margin ) \u003E\n\t\t\t\tTopLogitAbsoluteTolerance )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] Minimum generation margin expected step=\u0022 \u002B\n\t\t\t\t$\u0022{reference.MinimumMarginStep} value={reference.MinimumTop1Top2Margin:G9}, \u0022 \u002B\n\t\t\t\t$\u0022actual step={minimum.Step} value={minimum.Top1Top2Margin:G9}.\u0022 );\n\t\t}\n\n\t\tLlmLog.Info(\n\t\t\t\u0022PARITY\u0022,\n\t\t\t$\u0022greedy_sequence generated={generation.GeneratedTokenIds.Length} \u0022 \u002B\n\t\t\t$\u0022matched={generation.GeneratedTokenIds.Length} ids=\u0022 \u002B\n\t\t\t$\u0022[{string.Join( \u0022,\u0022, generation.GeneratedTokenIds )}] \u0022 \u002B\n\t\t\t$\u0022text=\u0027{EscapeVisible( generation.GeneratedText )}\u0027 stop={generation.StopReason} PASS\u0022 );\n\t\tLlmLog.Info(\n\t\t\t\u0022GEN\u0022,\n\t\t\t$\u0022minimum_margin={minimum.Top1Top2Margin:G9} step={minimum.Step} \u0022 \u002B\n\t\t\t$\u0022first_token_ms={generation.Steps[0].StepMilliseconds:N4} \u0022 \u002B\n\t\t\t$\u0022last_token_ms={generation.Steps[^1].StepMilliseconds:N4} \u0022 \u002B\n\t\t\t$\u0022total_forward_ms={generation.TotalForwardMilliseconds:N4} \u0022 \u002B\n\t\t\t$\u0022total_generation_ms={generation.TotalGenerationMilliseconds:N4} \u0022 \u002B\n\t\t\t$\u0022average_ms_per_token=\u0022 \u002B\n\t\t\t$\u0022{generation.TotalGenerationMilliseconds / generation.Steps.Length:N4} \u0022 \u002B\n\t\t\t$\u0022tokens_per_second=\u0022 \u002B\n\t\t\t$\u0022{generation.Steps.Length * 1000.0 / generation.TotalGenerationMilliseconds:N4} \u0022 \u002B\n\t\t\t\u0022kv_cache=false PASS\u0022 );\n\n\t\treturn new GreedyGenerationParityResult\n\t\t{\n\t\t\tGeneration = generation,\n\t\t\tLaterStepLogitComparison = laterComparison,\n\t\t\tLaterStepLogitIndex = reference.LaterFullLogitReference.Step,\n\t\t\tMinimumMargin = minimum.Top1Top2Margin,\n\t\t\tMinimumMarginStep = minimum.Step,\n\t\t\tValidationHarnessMilliseconds = validationTimer.ElapsedMilliSeconds\n\t\t};\n\t}\n\n\tprivate static void ValidateReference(\n\t\tTinyStoriesConfig config,\n\t\tGpt2ByteBpeTokenizer tokenizer,\n\t\tLlmReferenceData historical,\n\t\tGreedyGenerationReferenceDocument reference )\n\t{\n\t\tRequireExactArray( \u0022reference prompt IDs\u0022, historical.InputTokenIds, reference.InputTokenIds );\n\t\tRequireExactArray(\n\t\t\t\u0022historical generated IDs\u0022, historical.GeneratedTokenIds,\n\t\t\treference.GeneratedTokenIds );\n\t\tif ( historical.Prompt != reference.Prompt ||\n\t\t\thistorical.GeneratedText != reference.GeneratedText )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\u0022[LLM:ERROR] Compact generation reference differs from historical reference.json.\u0022 );\n\t\t}\n\t\tif ( reference.EosTokenId != config.EosTokenId ||\n\t\t\treference.BosTokenId != config.BosTokenId )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] Generation special IDs expected BOS/EOS \u0022 \u002B\n\t\t\t\t$\u0022{config.BosTokenId}/{config.EosTokenId}, found \u0022 \u002B\n\t\t\t\t$\u0022{reference.BosTokenId}/{reference.EosTokenId}.\u0022 );\n\t\t}\n\t\tif ( reference.DoSample || reference.BosAutomaticallyAdded ||\n\t\t\treference.EosAutomaticallyAdded || !reference.RawLogitsUsedForArgmax ||\n\t\t\t!reference.CachedAndUncachedSequencesMatch || reference.CsharpBaselineUseCache ||\n\t\t\treference.LogitsProcessors is null || reference.LogitsProcessors.Length != 0 ||\n\t\t\treference.ArgmaxTieBreak != \u0022first (lowest) vocabulary index\u0022 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\u0022[LLM:ERROR] Generation reference does not describe raw, uncached, \u0022 \u002B\n\t\t\t\t\u0022deterministic first-index argmax semantics.\u0022 );\n\t\t}\n\t\tif ( reference.MaxNewTokens \u003C= 0 || reference.Steps is null ||\n\t\t\treference.Steps.Length != reference.GeneratedTokenIds.Length ||\n\t\t\treference.Steps.Length \u003E reference.MaxNewTokens )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\u0022[LLM:ERROR] Generation reference step/token counts are inconsistent.\u0022 );\n\t\t}\n\t\tif ( tokenizer.Decode( reference.GeneratedTokenIds ) != reference.GeneratedText )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\u0022[LLM:ERROR] C# tokenizer cannot reproduce Python generated text from \u0022 \u002B\n\t\t\t\t\u0022the authoritative token sequence.\u0022 );\n\t\t}\n\t\tif ( reference.LaterFullLogitReference is null ||\n\t\t\treference.LaterFullLogitReference.Step \u003C 1 ||\n\t\t\treference.LaterFullLogitReference.Step \u003E= reference.Steps.Length ||\n\t\t\treference.LaterFullLogitReference.Elements != config.VocabularySize ||\n\t\t\treference.LaterFullLogitReference.Bytes != config.VocabularySize * sizeof( float ) )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\u0022[LLM:ERROR] Later generation full-logit reference metadata is invalid.\u0022 );\n\t\t}\n\t}\n\n\tprivate static void ValidateStepContext(\n\t\tGreedyGenerationReferenceStep expected,\n\t\tint[] actualInput )\n\t{\n\t\tif ( expected.Step \u003C 0 || expected.InputTokenIds is null ||\n\t\t\texpected.InputSequenceLength != expected.InputTokenIds.Length ||\n\t\t\texpected.NewTokenPosition != expected.InputSequenceLength )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] Generation reference step {expected.Step} context metadata is invalid.\u0022 );\n\t\t}\n\t\tRequireExactArray( $\u0022generation step {expected.Step} input\u0022, expected.InputTokenIds, actualInput );\n\t}\n\n\tprivate static void ValidateTopFive(\n\t\tGpt2ByteBpeTokenizer tokenizer,\n\t\tGreedyGenerationReferenceStep expected,\n\t\tGreedyGenerationStepResult actual )\n\t{\n\t\tif ( expected.TopFive is null || expected.TopFive.Length != 5 ||\n\t\t\tactual.TopFive is null || actual.TopFive.Length != 5 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] Generation step {actual.Step} requires five top-logit entries.\u0022 );\n\t\t}\n\t\tfor ( int rank = 0; rank \u003C 5; rank\u002B\u002B )\n\t\t{\n\t\t\tForwardReferenceTopLogit expectedRank = expected.TopFive[rank];\n\t\t\tLogitRank actualRank = actual.TopFive[rank];\n\t\t\tstring decoded = tokenizer.Decode( new[] { actualRank.TokenId } );\n\t\t\tif ( expectedRank.Rank != actualRank.Rank ||\n\t\t\t\texpectedRank.TokenId != actualRank.TokenId ||\n\t\t\t\texpectedRank.DecodedToken != decoded ||\n\t\t\t\tMath.Abs( expectedRank.Logit - actualRank.Logit ) \u003E TopLogitAbsoluteTolerance )\n\t\t\t{\n\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t$\u0022[LLM:ERROR] Generation step {actual.Step} top-5 rank {rank \u002B 1} \u0022 \u002B\n\t\t\t\t\t$\u0022expected token={expectedRank.TokenId} logit={expectedRank.Logit:G9} \u0022 \u002B\n\t\t\t\t\t$\u0022decoded=\u0027{EscapeVisible( expectedRank.DecodedToken )}\u0027, actual \u0022 \u002B\n\t\t\t\t\t$\u0022token={actualRank.TokenId} logit={actualRank.Logit:G9} \u0022 \u002B\n\t\t\t\t\t$\u0022decoded=\u0027{EscapeVisible( decoded )}\u0027.\u0022 );\n\t\t\t}\n\t\t}\n\t}\n\n\tprivate static void LogMismatch(\n\t\tGpt2ByteBpeTokenizer tokenizer,\n\t\tGreedyGenerationReferenceStep expected,\n\t\tGreedyGenerationStepObservation observation )\n\t{\n\t\tGreedyGenerationStepResult actual = observation.Step;\n\t\tLlmLog.Error(\n\t\t\t$\u0022Greedy mismatch step={actual.Step} context={actual.InputSequenceLength} \u0022 \u002B\n\t\t\t$\u0022input=[{string.Join( \u0022,\u0022, observation.InputTokenIds )}] \u0022 \u002B\n\t\t\t$\u0022expected={expected.ExpectedNextTokenId} \u0022 \u002B\n\t\t\t$\u0022expected_piece=\u0027{EscapeVisible( expected.DecodedToken )}\u0027 actual={actual.TokenId} \u0022 \u002B\n\t\t\t$\u0022actual_piece=\u0027{EscapeVisible( tokenizer.Decode( new[] { actual.TokenId } ) )}\u0027 \u0022 \u002B\n\t\t\t$\u0022expected_margin={expected.Top1Top2Margin:G9} \u0022 \u002B\n\t\t\t$\u0022actual_margin={actual.Top1Top2Margin:G9}.\u0022 );\n\t\tLlmLog.Error(\n\t\t\t$\u0022Python top5=[{string.Join( \u0022,\u0022, expected.TopFive.Select( item =\u003E $\u0022{item.TokenId}:{item.Logit:G9}\u0022 ) )}] \u0022 \u002B\n\t\t\t$\u0022C# top5=[{string.Join( \u0022,\u0022, actual.TopFive.Select( item =\u003E $\u0022{item.TokenId}:{item.Logit:G9}\u0022 ) )}]. \u0022 \u002B\n\t\t\t\u0022Generate a full Python logit reference for this step only before changing tolerances.\u0022 );\n\t}\n\n\tprivate static void RequireExactArray( string label, int[] expected, int[] actual )\n\t{\n\t\tif ( expected is null || actual is null || expected.Length != actual.Length )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {label} expected length {expected?.Length ?? -1}, \u0022 \u002B\n\t\t\t\t$\u0022actual {actual?.Length ?? -1}.\u0022 );\n\t\t}\n\t\tfor ( int index = 0; index \u003C expected.Length; index\u002B\u002B )\n\t\t{\n\t\t\tif ( expected[index] != actual[index] )\n\t\t\t{\n\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t$\u0022[LLM:ERROR] {label} mismatch at index {index}: \u0022 \u002B\n\t\t\t\t\t$\u0022expected={expected[index]}, actual={actual[index]}.\u0022 );\n\t\t\t}\n\t\t}\n\t}\n\n\tprivate static string EscapeVisible( string value )\n\t{\n\t\treturn (value ?? \u0022\u003Cnull\u003E\u0022)\n\t\t\t.Replace( \u0022\\\\\u0022, \u0022\\\\\\\\\u0022 )\n\t\t\t.Replace( \u0022\\r\u0022, \u0022\\\\r\u0022 )\n\t\t\t.Replace( \u0022\\n\u0022, \u0022\\\\n\u0022 )\n\t\t\t.Replace( \u0022\\t\u0022, \u0022\\\\t\u0022 )\n\t\t\t.Replace( \u0022\u0027\u0022, \u0022\\\\\u0027\u0022 );\n\t}\n}\n"},{"Ident":"jeffskitchen.llm_poc","Path":"Llm/LlmLog.cs","FileName":"LlmLog.cs","PackageType":"game","CodeKind":"Game","AssetVersionId":342521,"Code":"namespace LlmPoc.Llm;\n\npublic static class LlmLog\n{\n\tpublic static bool TraceEnabled { get; set; }\n\n\tpublic static void Info( string category, string message )\n\t{\n\t\tLog.Info( $\u0022[LLM:{category}] {message}\u0022 );\n\t}\n\n\tpublic static void Trace( string category, string message )\n\t{\n\t\tif ( TraceEnabled )\n\t\t{\n\t\t\tLog.Info( $\u0022[LLM:{category}] {message}\u0022 );\n\t\t}\n\t}\n\n\tpublic static void Warning( string category, string message )\n\t{\n\t\tLog.Warning( $\u0022[LLM:{category}] {message}\u0022 );\n\t}\n\n\tpublic static void Error( string message )\n\t{\n\t\tLog.Error( $\u0022[LLM:ERROR] {message}\u0022 );\n\t}\n}\n"},{"Ident":"jeffskitchen.llm_poc","Path":"Llm/TensorDiagnostics.cs","FileName":"TensorDiagnostics.cs","PackageType":"game","CodeKind":"Game","AssetVersionId":342521,"Code":"using System.Text;\n\nnamespace LlmPoc.Llm;\n\npublic sealed class TensorSummary\n{\n\tpublic string Name { get; init; }\n\tpublic string Shape { get; init; }\n\tpublic long ElementCount { get; init; }\n\tpublic float Minimum { get; init; }\n\tpublic float Maximum { get; init; }\n\tpublic double Mean { get; init; }\n\tpublic double StandardDeviation { get; init; }\n\tpublic double Rms { get; init; }\n\tpublic int NaNCount { get; init; }\n\tpublic int PositiveInfinityCount { get; init; }\n\tpublic int NegativeInfinityCount { get; init; }\n\tpublic string FirstValues { get; init; }\n\tpublic string LastValues { get; init; }\n\tpublic ulong Checksum { get; init; }\n\n\tpublic bool IsFinite =\u003E NaNCount == 0 \u0026\u0026 PositiveInfinityCount == 0 \u0026\u0026 NegativeInfinityCount == 0;\n\tpublic long FiniteCount =\u003E ElementCount - NaNCount - PositiveInfinityCount - NegativeInfinityCount;\n\n\tpublic override string ToString()\n\t{\n\t\treturn $\u0022name={Name} shape={Shape} count={ElementCount:N0} \u0022 \u002B\n\t\t\t$\u0022min={Minimum:G9} max={Maximum:G9} mean={Mean:G12} \u0022 \u002B\n\t\t\t$\u0022std={StandardDeviation:G12} rms={Rms:G12} finite={FiniteCount:N0}/{ElementCount:N0} \u0022 \u002B\n\t\t\t$\u0022nan={NaNCount} \u002Binf={PositiveInfinityCount} -inf={NegativeInfinityCount} \u0022 \u002B\n\t\t\t$\u0022first={FirstValues} last={LastValues} fnv1a64={Checksum:X16}\u0022;\n\t}\n}\n\npublic sealed class NumericComparison\n{\n\tpublic int ElementCount { get; init; }\n\tpublic double MaximumAbsoluteError { get; init; }\n\tpublic double MeanAbsoluteError { get; init; }\n\tpublic double MaximumRelativeError { get; init; }\n\tpublic int MaximumErrorIndex { get; init; }\n\tpublic float ExpectedAtMaximumError { get; init; }\n\tpublic float ActualAtMaximumError { get; init; }\n\tpublic int FirstFailingIndex { get; init; }\n\tpublic float ExpectedAtFirstFailure { get; init; }\n\tpublic float ActualAtFirstFailure { get; init; }\n\tpublic double AbsoluteTolerance { get; init; }\n\tpublic double RelativeTolerance { get; init; }\n\tpublic bool Passed { get; init; }\n\n\tpublic override string ToString()\n\t{\n\t\tstring failure = FirstFailingIndex \u003C 0\n\t\t\t? \u0022\u0022\n\t\t\t: $\u0022 first_fail_index={FirstFailingIndex} \u0022 \u002B\n\t\t\t\t$\u0022first_expected={ExpectedAtFirstFailure:G9} first_actual={ActualAtFirstFailure:G9}\u0022;\n\t\treturn $\u0022count={ElementCount:N0} max_abs={MaximumAbsoluteError:G12} \u0022 \u002B\n\t\t\t$\u0022mean_abs={MeanAbsoluteError:G12} max_rel={MaximumRelativeError:G12} \u0022 \u002B\n\t\t\t$\u0022max_index={MaximumErrorIndex} expected={ExpectedAtMaximumError:G9} \u0022 \u002B\n\t\t\t$\u0022actual={ActualAtMaximumError:G9} abs_tol={AbsoluteTolerance:G6} \u0022 \u002B\n\t\t\t$\u0022rel_tol={RelativeTolerance:G6}{failure} {(Passed ? \u0022PASS\u0022 : \u0022FAIL\u0022)}\u0022;\n\t}\n}\n\npublic static class TensorDiagnostics\n{\n\tprivate const ulong FnvOffsetBasis = 14695981039346656037UL;\n\tprivate const ulong FnvPrime = 1099511628211UL;\n\n\tpublic static TensorSummary Summarize( Tensor tensor, int edgeCount = 4 )\n\t{\n\t\tif ( tensor is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( tensor ) );\n\t\t}\n\t\treturn Summarize( tensor.Name, tensor.ShapeText, tensor.Data, edgeCount );\n\t}\n\n\tpublic static TensorSummary Summarize(\n\t\tstring name,\n\t\tstring shape,\n\t\tReadOnlySpan\u003Cfloat\u003E values,\n\t\tint edgeCount = 4 )\n\t{\n\t\tif ( edgeCount \u003C 0 || edgeCount \u003E 32 )\n\t\t{\n\t\t\tthrow new ArgumentOutOfRangeException(\n\t\t\t\tnameof( edgeCount ), edgeCount, \u0022[LLM:ERROR] Edge count must be between 0 and 32.\u0022 );\n\t\t}\n\t\tif ( values.Length == 0 )\n\t\t{\n\t\t\tthrow new ArgumentException( $\u0022[LLM:ERROR] {name} cannot be summarized because it is empty.\u0022 );\n\t\t}\n\n\t\tfloat minimum = float.PositiveInfinity;\n\t\tfloat maximum = float.NegativeInfinity;\n\t\tdouble sum = 0;\n\t\tdouble sumSquares = 0;\n\t\tint finiteCount = 0;\n\t\tint nanCount = 0;\n\t\tint positiveInfinityCount = 0;\n\t\tint negativeInfinityCount = 0;\n\t\tulong checksum = FnvOffsetBasis;\n\n\t\tfor ( int index = 0; index \u003C values.Length; index\u002B\u002B )\n\t\t{\n\t\t\tfloat value = values[index];\n\t\t\tuint bits = unchecked( (uint)BitConverter.SingleToInt32Bits( value ) );\n\t\t\tchecksum = HashByte( checksum, (byte)bits );\n\t\t\tchecksum = HashByte( checksum, (byte)(bits \u003E\u003E 8) );\n\t\t\tchecksum = HashByte( checksum, (byte)(bits \u003E\u003E 16) );\n\t\t\tchecksum = HashByte( checksum, (byte)(bits \u003E\u003E 24) );\n\n\t\t\tif ( float.IsNaN( value ) )\n\t\t\t{\n\t\t\t\tnanCount\u002B\u002B;\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif ( float.IsPositiveInfinity( value ) )\n\t\t\t{\n\t\t\t\tpositiveInfinityCount\u002B\u002B;\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif ( float.IsNegativeInfinity( value ) )\n\t\t\t{\n\t\t\t\tnegativeInfinityCount\u002B\u002B;\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tminimum = Math.Min( minimum, value );\n\t\t\tmaximum = Math.Max( maximum, value );\n\t\t\tsum \u002B= value;\n\t\t\tsumSquares \u002B= (double)value * value;\n\t\t\tfiniteCount\u002B\u002B;\n\t\t}\n\n\t\tdouble mean = finiteCount == 0 ? double.NaN : sum / finiteCount;\n\t\tdouble rms = finiteCount == 0 ? double.NaN : Math.Sqrt( sumSquares / finiteCount );\n\t\tdouble variance = finiteCount == 0 ? double.NaN : Math.Max( 0, sumSquares / finiteCount - mean * mean );\n\n\t\treturn new TensorSummary\n\t\t{\n\t\t\tName = name,\n\t\t\tShape = shape,\n\t\t\tElementCount = values.Length,\n\t\t\tMinimum = finiteCount == 0 ? float.NaN : minimum,\n\t\t\tMaximum = finiteCount == 0 ? float.NaN : maximum,\n\t\t\tMean = mean,\n\t\t\tStandardDeviation = Math.Sqrt( variance ),\n\t\t\tRms = rms,\n\t\t\tNaNCount = nanCount,\n\t\t\tPositiveInfinityCount = positiveInfinityCount,\n\t\t\tNegativeInfinityCount = negativeInfinityCount,\n\t\t\tFirstValues = FormatEdge( values, 0, Math.Min( edgeCount, values.Length ) ),\n\t\t\tLastValues = FormatEdge( values, Math.Max( 0, values.Length - edgeCount ), Math.Min( edgeCount, values.Length ) ),\n\t\t\tChecksum = checksum\n\t\t};\n\t}\n\n\tpublic static NumericComparison Compare(\n\t\tReadOnlySpan\u003Cfloat\u003E expected,\n\t\tReadOnlySpan\u003Cfloat\u003E actual,\n\t\tdouble absoluteTolerance = 1e-4,\n\t\tdouble relativeTolerance = 1e-3 )\n\t{\n\t\tif ( expected.Length != actual.Length )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t$\u0022[LLM:ERROR] Parity comparison shape mismatch: expected {expected.Length:N0} \u0022 \u002B\n\t\t\t\t$\u0022elements, actual {actual.Length:N0}.\u0022 );\n\t\t}\n\t\tif ( expected.Length == 0 )\n\t\t{\n\t\t\tthrow new ArgumentException( \u0022[LLM:ERROR] Parity comparison cannot use empty arrays.\u0022 );\n\t\t}\n\t\tif ( absoluteTolerance \u003C 0 || relativeTolerance \u003C 0 )\n\t\t{\n\t\t\tthrow new ArgumentOutOfRangeException(\n\t\t\t\tnameof( absoluteTolerance ), \u0022[LLM:ERROR] Parity tolerances cannot be negative.\u0022 );\n\t\t}\n\n\t\tdouble maxAbsolute = -1;\n\t\tdouble maxRelative = 0;\n\t\tdouble absoluteSum = 0;\n\t\tint maxIndex = 0;\n\t\tint firstFailingIndex = -1;\n\t\tbool passed = true;\n\n\t\tfor ( int index = 0; index \u003C expected.Length; index\u002B\u002B )\n\t\t{\n\t\t\tfloat expectedValue = expected[index];\n\t\t\tfloat actualValue = actual[index];\n\t\t\tbool finite = float.IsFinite( expectedValue ) \u0026\u0026 float.IsFinite( actualValue );\n\t\t\tif ( !finite )\n\t\t\t{\n\t\t\t\tpassed = false;\n\t\t\t\tif ( firstFailingIndex \u003C 0 )\n\t\t\t\t{\n\t\t\t\t\tfirstFailingIndex = index;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tdouble absolute = Math.Abs( (double)actualValue - expectedValue );\n\t\t\tdouble scale = Math.Max( Math.Abs( expectedValue ), 1e-12 );\n\t\t\tdouble relative = absolute / scale;\n\t\t\tabsoluteSum \u002B= absolute;\n\t\t\tmaxRelative = Math.Max( maxRelative, relative );\n\t\t\tif ( absolute \u003E maxAbsolute )\n\t\t\t{\n\t\t\t\tmaxAbsolute = absolute;\n\t\t\t\tmaxIndex = index;\n\t\t\t}\n\n\t\t\tif ( absolute \u003E absoluteTolerance \u002B relativeTolerance * Math.Abs( expectedValue ) )\n\t\t\t{\n\t\t\t\tpassed = false;\n\t\t\t\tif ( firstFailingIndex \u003C 0 )\n\t\t\t\t{\n\t\t\t\t\tfirstFailingIndex = index;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn new NumericComparison\n\t\t{\n\t\t\tElementCount = expected.Length,\n\t\t\tMaximumAbsoluteError = maxAbsolute,\n\t\t\tMeanAbsoluteError = absoluteSum / expected.Length,\n\t\t\tMaximumRelativeError = maxRelative,\n\t\t\tMaximumErrorIndex = maxIndex,\n\t\t\tExpectedAtMaximumError = expected[maxIndex],\n\t\t\tActualAtMaximumError = actual[maxIndex],\n\t\t\tFirstFailingIndex = firstFailingIndex,\n\t\t\tExpectedAtFirstFailure = firstFailingIndex \u003C 0 ? 0 : expected[firstFailingIndex],\n\t\t\tActualAtFirstFailure = firstFailingIndex \u003C 0 ? 0 : actual[firstFailingIndex],\n\t\t\tAbsoluteTolerance = absoluteTolerance,\n\t\t\tRelativeTolerance = relativeTolerance,\n\t\t\tPassed = passed\n\t\t};\n\t}\n\n\tprivate static ulong HashByte( ulong hash, byte value )\n\t{\n\t\treturn (hash ^ value) * FnvPrime;\n\t}\n\n\tprivate static string FormatEdge( ReadOnlySpan\u003Cfloat\u003E values, int start, int count )\n\t{\n\t\tStringBuilder builder = new();\n\t\tbuilder.Append( \u0027[\u0027 );\n\t\tfor ( int index = 0; index \u003C count; index\u002B\u002B )\n\t\t{\n\t\t\tif ( index \u003E 0 )\n\t\t\t{\n\t\t\t\tbuilder.Append( \u0027,\u0027 );\n\t\t\t}\n\t\t\tbuilder.Append( values[start \u002B index].ToString( \u0022G9\u0022 ) );\n\t\t}\n\t\tbuilder.Append( \u0027]\u0027 );\n\t\treturn builder.ToString();\n\t}\n}\n"},{"Ident":"jeffskitchen.llm_poc","Path":"Llm/TinyStoriesForwardStages.cs","FileName":"TinyStoriesForwardStages.cs","PackageType":"game","CodeKind":"Game","AssetVersionId":342521,"Code":"namespace LlmPoc.Llm;\n\npublic static class TinyStoriesForwardStages\n{\n\tpublic const string TokenEmbeddingName = \u0022transformer.wte.weight\u0022;\n\tpublic const string PositionEmbeddingName = \u0022transformer.wpe.weight\u0022;\n\tpublic const string Layer0Ln1WeightName = \u0022transformer.h.0.ln_1.weight\u0022;\n\tpublic const string Layer0Ln1BiasName = \u0022transformer.h.0.ln_1.bias\u0022;\n\tpublic const string Layer0QWeightName = \u0022transformer.h.0.attn.attention.q_proj.weight\u0022;\n\tpublic const string Layer0KWeightName = \u0022transformer.h.0.attn.attention.k_proj.weight\u0022;\n\tpublic const string Layer0VWeightName = \u0022transformer.h.0.attn.attention.v_proj.weight\u0022;\n\tpublic const string Layer0AttentionOutProjectionWeightName =\n\t\t\u0022transformer.h.0.attn.attention.out_proj.weight\u0022;\n\tpublic const string Layer0AttentionOutProjectionBiasName =\n\t\t\u0022transformer.h.0.attn.attention.out_proj.bias\u0022;\n\tpublic const string Layer0Ln2WeightName = \u0022transformer.h.0.ln_2.weight\u0022;\n\tpublic const string Layer0Ln2BiasName = \u0022transformer.h.0.ln_2.bias\u0022;\n\tpublic const string Layer0MlpFcWeightName = \u0022transformer.h.0.mlp.c_fc.weight\u0022;\n\tpublic const string Layer0MlpFcBiasName = \u0022transformer.h.0.mlp.c_fc.bias\u0022;\n\tpublic const string Layer0MlpProjWeightName = \u0022transformer.h.0.mlp.c_proj.weight\u0022;\n\tpublic const string Layer0MlpProjBiasName = \u0022transformer.h.0.mlp.c_proj.bias\u0022;\n\tpublic const float GeluNewTanhCoefficient = 0.7978845608028654f;\n\tpublic const float GeluNewCubicCoefficient = 0.044715f;\n\tpublic const string GeluNewFormula =\n\t\t\u00220.5*x*(1.0\u002Btanh(sqrt(2.0/pi)*(x\u002B0.044715*pow(x,3.0))))\u0022;\n\n\tpublic static Tensor CombineEmbeddings(\n\t\tSboxLlmModel model,\n\t\tTinyStoriesConfig config,\n\t\tIReadOnlyList\u003Cint\u003E tokenIds,\n\t\tbool logDiagnostics = true )\n\t{\n\t\tif ( model is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( model ) );\n\t\t}\n\t\tif ( config is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( config ) );\n\t\t}\n\t\tif ( tokenIds is null || tokenIds.Count == 0 )\n\t\t{\n\t\t\tthrow new ArgumentException( \u0022[LLM:ERROR] Embedding input token IDs cannot be empty.\u0022 );\n\t\t}\n\t\tif ( tokenIds.Count \u003E config.MaximumPositions )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] Embedding sequence length {tokenIds.Count} exceeds \u0022 \u002B\n\t\t\t\t$\u0022maximum positions {config.MaximumPositions}.\u0022 );\n\t\t}\n\n\t\tTensor tokenEmbedding = model.GetRequiredTensor( TokenEmbeddingName );\n\t\tTensor positionEmbedding = model.GetRequiredTensor( PositionEmbeddingName );\n\t\ttokenEmbedding.RequireShape( config.VocabularySize, config.HiddenSize );\n\t\tpositionEmbedding.RequireShape( config.MaximumPositions, config.HiddenSize );\n\n\t\tif ( tokenIds.Count \u003E int.MaxValue / config.HiddenSize )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] Embedding output shape [{tokenIds.Count},{config.HiddenSize}] \u0022 \u002B\n\t\t\t\t\u0022exceeds the managed array limit.\u0022 );\n\t\t}\n\n\t\tif ( logDiagnostics )\n\t\t{\n\t\t\tLlmLog.Info(\n\t\t\t\t\u0022EMBED\u0022,\n\t\t\t\t$\u0022input shape=[{tokenIds.Count}] tokens=[{string.Join( \u0022,\u0022, tokenIds )}] \u0022 \u002B\n\t\t\t\t$\u0022positions=[0..{tokenIds.Count - 1}] token_tensor={TokenEmbeddingName}{tokenEmbedding.ShapeText} \u0022 \u002B\n\t\t\t\t$\u0022position_tensor={PositionEmbeddingName}{positionEmbedding.ShapeText}\u0022 );\n\t\t}\n\n\t\tint hiddenSize = config.HiddenSize;\n\t\tfloat[] output = new float[tokenIds.Count * hiddenSize];\n\t\tfor ( int position = 0; position \u003C tokenIds.Count; position\u002B\u002B )\n\t\t{\n\t\t\tint tokenId = tokenIds[position];\n\t\t\tif ( tokenId \u003C 0 || tokenId \u003E= config.VocabularySize )\n\t\t\t{\n\t\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t\t$\u0022[LLM:ERROR] Embedding token ID {tokenId} at sequence index {position} \u0022 \u002B\n\t\t\t\t\t$\u0022is outside [0,{config.VocabularySize}).\u0022 );\n\t\t\t}\n\n\t\t\tint tokenRow = tokenId * hiddenSize;\n\t\t\tint positionRow = position * hiddenSize;\n\t\t\tint outputRow = position * hiddenSize;\n\t\t\tfor ( int hidden = 0; hidden \u003C hiddenSize; hidden\u002B\u002B )\n\t\t\t{\n\t\t\t\toutput[outputRow \u002B hidden] =\n\t\t\t\t\ttokenEmbedding.Data[tokenRow \u002B hidden] \u002B\n\t\t\t\t\tpositionEmbedding.Data[positionRow \u002B hidden];\n\t\t\t}\n\t\t}\n\n\t\treturn new Tensor(\n\t\t\t\u0022forward.combined_embedding\u0022,\n\t\t\tnew[] { tokenIds.Count, hiddenSize },\n\t\t\toutput );\n\t}\n\n\tpublic static Tensor ApplyLayer0Ln1(\n\t\tSboxLlmModel model,\n\t\tTinyStoriesConfig config,\n\t\tTensor input )\n\t{\n\t\treturn ApplyLayerNorm(\n\t\t\tmodel,\n\t\t\tconfig,\n\t\t\tinput,\n\t\t\tLayer0Ln1WeightName,\n\t\t\tLayer0Ln1BiasName,\n\t\t\t\u0022layer=0 ln=1\u0022,\n\t\t\t\u0022Layer 0 ln_1\u0022,\n\t\t\t\u0022forward.layer0.ln_1\u0022 );\n\t}\n\n\tpublic static Tensor ApplyLayer0Ln2(\n\t\tSboxLlmModel model,\n\t\tTinyStoriesConfig config,\n\t\tTensor input )\n\t{\n\t\treturn ApplyLayerNorm(\n\t\t\tmodel,\n\t\t\tconfig,\n\t\t\tinput,\n\t\t\tLayer0Ln2WeightName,\n\t\t\tLayer0Ln2BiasName,\n\t\t\t\u0022layer=0 ln=2\u0022,\n\t\t\t\u0022Layer 0 ln_2\u0022,\n\t\t\t\u0022forward.layer0.ln_2\u0022 );\n\t}\n\n\tpublic static Tensor ApplyLayerNorm(\n\t\tSboxLlmModel model,\n\t\tTinyStoriesConfig config,\n\t\tTensor input,\n\t\tint layerIndex,\n\t\tint normIndex,\n\t\tstring outputName,\n\t\tbool logDiagnostics = true )\n\t{\n\t\tif ( layerIndex \u003C 0 || layerIndex \u003E= config.LayerCount )\n\t\t{\n\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t$\u0022[LLM:ERROR] LayerNorm layer index {layerIndex} is outside \u0022 \u002B\n\t\t\t\t$\u0022[0,{config.LayerCount}).\u0022 );\n\t\t}\n\t\tif ( normIndex != 1 \u0026\u0026 normIndex != 2 )\n\t\t{\n\t\t\tthrow new ArgumentOutOfRangeException(\n\t\t\t\tnameof( normIndex ), normIndex,\n\t\t\t\t\u0022[LLM:ERROR] GPT-Neo transformer LayerNorm index must be 1 or 2.\u0022 );\n\t\t}\n\n\t\tstring prefix = $\u0022transformer.h.{layerIndex}.ln_{normIndex}\u0022;\n\t\treturn ApplyLayerNorm(\n\t\t\tmodel,\n\t\t\tconfig,\n\t\t\tinput,\n\t\t\t$\u0022{prefix}.weight\u0022,\n\t\t\t$\u0022{prefix}.bias\u0022,\n\t\t\t$\u0022layer={layerIndex} ln={normIndex}\u0022,\n\t\t\t$\u0022Layer {layerIndex} ln_{normIndex}\u0022,\n\t\t\toutputName,\n\t\t\tlogDiagnostics );\n\t}\n\n\tpublic static Tensor ApplyFinalLayerNorm(\n\t\tSboxLlmModel model,\n\t\tTinyStoriesConfig config,\n\t\tTensor input,\n\t\tstring outputName,\n\t\tbool logDiagnostics = true )\n\t{\n\t\treturn ApplyLayerNorm(\n\t\t\tmodel,\n\t\t\tconfig,\n\t\t\tinput,\n\t\t\t\u0022transformer.ln_f.weight\u0022,\n\t\t\t\u0022transformer.ln_f.bias\u0022,\n\t\t\t\u0022stage=final_ln\u0022,\n\t\t\t\u0022Final model LayerNorm\u0022,\n\t\t\toutputName,\n\t\t\tlogDiagnostics );\n\t}\n\n\tprivate static Tensor ApplyLayerNorm(\n\t\tSboxLlmModel model,\n\t\tTinyStoriesConfig config,\n\t\tTensor input,\n\t\tstring weightName,\n\t\tstring biasName,\n\t\tstring logLabel,\n\t\tstring errorLabel,\n\t\tstring outputName,\n\t\tbool logDiagnostics = true )\n\t{\n\t\tif ( model is null ) throw new ArgumentNullException( nameof( model ) );\n\t\tif ( config is null ) throw new ArgumentNullException( nameof( config ) );\n\t\tif ( input is null ) throw new ArgumentNullException( nameof( input ) );\n\t\tif ( input.Rank != 2 || input.Shape[0] \u003C= 0 || input.Shape[1] != config.HiddenSize )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {errorLabel} input expected shape [sequence,{config.HiddenSize}] \u0022 \u002B\n\t\t\t\t$\u0022with a positive sequence length, found {input.ShapeText}.\u0022 );\n\t\t}\n\n\t\tTensor weight = model.GetRequiredTensor( weightName );\n\t\tTensor bias = model.GetRequiredTensor( biasName );\n\t\tweight.RequireShape( config.HiddenSize );\n\t\tbias.RequireShape( config.HiddenSize );\n\t\tif ( !(config.LayerNormEpsilon \u003E 0) || !float.IsFinite( config.LayerNormEpsilon ) )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {errorLabel} epsilon must be positive and finite, \u0022 \u002B\n\t\t\t\t$\u0022found {config.LayerNormEpsilon:G9}.\u0022 );\n\t\t}\n\n\t\tint sequenceLength = input.Shape[0];\n\t\tint hiddenSize = config.HiddenSize;\n\t\tfloat[] output = new float[input.Data.Length];\n\t\tif ( logDiagnostics )\n\t\t{\n\t\t\tLlmLog.Info(\n\t\t\t\t\u0022LN\u0022,\n\t\t\t\t$\u0022{logLabel} input_shape={input.ShapeText} expected_shape=[{sequenceLength},{hiddenSize}] \u0022 \u002B\n\t\t\t\t$\u0022weight={weightName}{weight.ShapeText} bias={biasName}{bias.ShapeText} \u0022 \u002B\n\t\t\t\t$\u0022epsilon={config.LayerNormEpsilon:G9} variance=population(unbiased=false)\u0022 );\n\t\t}\n\n\t\tfor ( int token = 0; token \u003C sequenceLength; token\u002B\u002B )\n\t\t{\n\t\t\tint row = token * hiddenSize;\n\t\t\t// PyTorch 2.9.1\u0027s installed AVX2 LayerNorm kernel uses RowwiseMoments:\n\t\t\t// eight FP32 Welford lanes over this model\u0027s 64 hidden values, followed\n\t\t\t// by a left-to-right cascade of the lane moments. This scalar spelling\n\t\t\t// reproduces that numerical reduction order without introducing SIMD.\n\t\t\t(float mean, float variance) = ComputeAvx2RowwiseMoments64( input.Data, row );\n\t\t\tfloat denominator = MathF.Sqrt( variance \u002B config.LayerNormEpsilon );\n\t\t\tif ( !(denominator \u003E 0) || !float.IsFinite( denominator ) )\n\t\t\t{\n\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t$\u0022[LLM:ERROR] {errorLabel} token {token} produced invalid denominator \u0022 \u002B\n\t\t\t\t\t$\u0022{denominator:G9} from mean={mean:G9}, variance={variance:G9}, \u0022 \u002B\n\t\t\t\t\t$\u0022epsilon={config.LayerNormEpsilon:G9}.\u0022 );\n\t\t\t}\n\t\t\tfloat inverseStandardDeviation = 1.0f / denominator;\n\n\t\t\tfor ( int hidden = 0; hidden \u003C hiddenSize; hidden\u002B\u002B )\n\t\t\t{\n\t\t\t\tfloat normalized = (input.Data[row \u002B hidden] - mean) * inverseStandardDeviation;\n\t\t\t\toutput[row \u002B hidden] = normalized * weight.Data[hidden] \u002B bias.Data[hidden];\n\t\t\t}\n\n\t\t\tif ( logDiagnostics \u0026\u0026 token == 0 )\n\t\t\t{\n\t\t\t\tLlmLog.Trace(\n\t\t\t\t\t\u0022LN\u0022,\n\t\t\t\t\t$\u0022{logLabel} token=0 mean={mean:G12} variance={variance:G12} \u0022 \u002B\n\t\t\t\t\t$\u0022denominator={denominator:G12} inverse_std={inverseStandardDeviation:G12} \u0022 \u002B\n\t\t\t\t\t$\u0022output_first=[{output[row]:G9},{output[row \u002B 1]:G9},\u0022 \u002B\n\t\t\t\t\t$\u0022{output[row \u002B 2]:G9},{output[row \u002B 3]:G9}]\u0022 );\n\t\t\t}\n\t\t}\n\n\t\treturn new Tensor(\n\t\t\toutputName,\n\t\t\tnew[] { sequenceLength, hiddenSize },\n\t\t\toutput );\n\t}\n\n\tprivate static (float Mean, float Variance) ComputeAvx2RowwiseMoments64(\n\t\tfloat[] values,\n\t\tint rowOffset )\n\t{\n\t\tconst int laneCount = 8;\n\t\tconst int valuesPerLane = 8;\n\t\tfloat[] laneMeans = new float[laneCount];\n\t\tfloat[] laneMoment2 = new float[laneCount];\n\n\t\tfor ( int item = 0; item \u003C valuesPerLane; item\u002B\u002B )\n\t\t{\n\t\t\tfloat reciprocalCount = 1.0f / (item \u002B 1);\n\t\t\tint itemOffset = rowOffset \u002B item * laneCount;\n\t\t\tfor ( int lane = 0; lane \u003C laneCount; lane\u002B\u002B )\n\t\t\t{\n\t\t\t\tfloat value = values[itemOffset \u002B lane];\n\t\t\t\tfloat delta = value - laneMeans[lane];\n\t\t\t\tfloat meanIncrement = delta * reciprocalCount;\n\t\t\t\tlaneMeans[lane] \u002B= meanIncrement;\n\t\t\t\tfloat remainingDelta = value - laneMeans[lane];\n\t\t\t\tfloat momentIncrement = delta * remainingDelta;\n\t\t\t\tlaneMoment2[lane] \u002B= momentIncrement;\n\t\t\t}\n\t\t}\n\n\t\tint accumulatedCount = 0;\n\t\tfloat mean = 0.0f;\n\t\tfloat moment2 = 0.0f;\n\t\tfor ( int lane = 0; lane \u003C laneCount; lane\u002B\u002B )\n\t\t{\n\t\t\tint combinedCount = accumulatedCount \u002B valuesPerLane;\n\t\t\tfloat contribution = (float)valuesPerLane / combinedCount;\n\t\t\tfloat delta = laneMeans[lane] - mean;\n\t\t\tfloat meanIncrement = contribution * delta;\n\t\t\tmean \u002B= meanIncrement;\n\n\t\t\tfloat deltaSquared = delta * delta;\n\t\t\tfloat weightedDelta = deltaSquared * contribution;\n\t\t\tweightedDelta *= accumulatedCount;\n\t\t\tfloat combinedMoment = laneMoment2[lane] \u002B weightedDelta;\n\t\t\tmoment2 \u002B= combinedMoment;\n\t\t\taccumulatedCount = combinedCount;\n\t\t}\n\n\t\treturn (mean, moment2 / 64.0f);\n\t}\n\n\tpublic static Tensor LinearNoBias( Tensor input, Tensor weight, string outputName )\n\t{\n\t\treturn Linear( input, weight, null, outputName );\n\t}\n\n\tpublic static Tensor LinearWithBias(\n\t\tTensor input,\n\t\tTensor weight,\n\t\tTensor bias,\n\t\tstring outputName )\n\t{\n\t\tif ( bias is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( bias ) );\n\t\t}\n\t\treturn Linear( input, weight, bias, outputName );\n\t}\n\n\tprivate static Tensor Linear(\n\t\tTensor input,\n\t\tTensor weight,\n\t\tTensor bias,\n\t\tstring outputName )\n\t{\n\t\tif ( input is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( input ) );\n\t\t}\n\t\tif ( weight is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( weight ) );\n\t\t}\n\t\tif ( string.IsNullOrWhiteSpace( outputName ) )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t\u0022[LLM:ERROR] Linear projection output name cannot be empty.\u0022,\n\t\t\t\tnameof( outputName ) );\n\t\t}\n\t\tif ( input.Rank != 2 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {outputName} linear input expected rank 2 \u0022 \u002B\n\t\t\t\t$\u0022[sequence,input_size], found {input.ShapeText}.\u0022 );\n\t\t}\n\t\tif ( weight.Rank != 2 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {outputName} weight \u0027{weight.Name}\u0027 expected rank 2 \u0022 \u002B\n\t\t\t\t$\u0022[output_size,input_size], found {weight.ShapeText}.\u0022 );\n\t\t}\n\n\t\tint sequenceLength = input.Shape[0];\n\t\tint inputSize = input.Shape[1];\n\t\tint outputSize = weight.Shape[0];\n\t\tif ( weight.Shape[1] != inputSize )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {outputName} incompatible linear shapes: input={input.ShapeText}, \u0022 \u002B\n\t\t\t\t$\u0022weight={weight.ShapeText}; weight input axis expected {inputSize}, \u0022 \u002B\n\t\t\t\t$\u0022found {weight.Shape[1]}.\u0022 );\n\t\t}\n\t\tif ( bias is not null )\n\t\t{\n\t\t\tif ( bias.Rank != 1 || bias.Shape[0] != outputSize )\n\t\t\t{\n\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t$\u0022[LLM:ERROR] {outputName} linear bias \u0027{bias.Name}\u0027 expected shape \u0022 \u002B\n\t\t\t\t\t$\u0022[{outputSize}], found {bias.ShapeText}.\u0022 );\n\t\t\t}\n\t\t}\n\t\tif ( sequenceLength \u003E int.MaxValue / outputSize )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {outputName} output shape [{sequenceLength},{outputSize}] \u0022 \u002B\n\t\t\t\t\u0022exceeds the managed array limit.\u0022 );\n\t\t}\n\n\t\tfloat[] output = new float[sequenceLength * outputSize];\n\t\tfor ( int token = 0; token \u003C sequenceLength; token\u002B\u002B )\n\t\t{\n\t\t\tint inputRow = token * inputSize;\n\t\t\tint outputRow = token * outputSize;\n\t\t\tfor ( int outputFeature = 0; outputFeature \u003C outputSize; outputFeature\u002B\u002B )\n\t\t\t{\n\t\t\t\tint weightRow = outputFeature * inputSize;\n\t\t\t\t// The installed PyTorch 2.9.1 CPU Linear path was independently\n\t\t\t\t// checked against the live module outputs. For this model it is\n\t\t\t\t// bit-identical to a left-to-right FP32 fused multiply-add reduction.\n\t\t\t\t// The small attention matmuls intentionally retain their separately\n\t\t\t\t// validated non-fused multiply/add loops.\n\t\t\t\tfloat sum = 0.0f;\n\t\t\t\tfor ( int inputFeature = 0; inputFeature \u003C inputSize; inputFeature\u002B\u002B )\n\t\t\t\t{\n\t\t\t\t\tsum = MathF.FusedMultiplyAdd(\n\t\t\t\t\t\tinput.Data[inputRow \u002B inputFeature],\n\t\t\t\t\t\tweight.Data[weightRow \u002B inputFeature],\n\t\t\t\t\t\tsum );\n\t\t\t\t}\n\t\t\t\toutput[outputRow \u002B outputFeature] =\n\t\t\t\t\tbias is null ? sum : sum \u002B bias.Data[outputFeature];\n\t\t\t}\n\t\t}\n\n\t\treturn new Tensor(\n\t\t\toutputName,\n\t\t\tnew[] { sequenceLength, outputSize },\n\t\t\toutput );\n\t}\n\n\tpublic static string DescribeLinearDotProduct(\n\t\tTensor input,\n\t\tTensor weight,\n\t\tint token,\n\t\tint outputFeature,\n\t\tfloat expected,\n\t\tfloat actual,\n\t\tTensor bias = null )\n\t{\n\t\tif ( input is null || weight is null || input.Rank != 2 || weight.Rank != 2 ||\n\t\t\tweight.Shape[1] != input.Shape[1] )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\u0022[LLM:ERROR] Dot-product diagnostic requires compatible rank-2 input and weight tensors.\u0022 );\n\t\t}\n\t\tif ( bias is not null \u0026\u0026 (bias.Rank != 1 || bias.Shape[0] != weight.Shape[0]) )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] Dot-product diagnostic bias \u0027{bias.Name}\u0027 expected shape \u0022 \u002B\n\t\t\t\t$\u0022[{weight.Shape[0]}], found {bias.ShapeText}.\u0022 );\n\t\t}\n\t\tif ( token \u003C 0 || token \u003E= input.Shape[0] ||\n\t\t\toutputFeature \u003C 0 || outputFeature \u003E= weight.Shape[0] )\n\t\t{\n\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t$\u0022[LLM:ERROR] Dot-product diagnostic index token={token}, \u0022 \u002B\n\t\t\t\t$\u0022output_feature={outputFeature} is outside input={input.ShapeText}, \u0022 \u002B\n\t\t\t\t$\u0022weight={weight.ShapeText}.\u0022 );\n\t\t}\n\n\t\tint inputSize = input.Shape[1];\n\t\tint inputRow = token * inputSize;\n\t\tint weightRow = outputFeature * inputSize;\n\t\tfloat sum = 0;\n\t\tfloat product0 = input.Data[inputRow] * weight.Data[weightRow];\n\t\tfloat product1 = input.Data[inputRow \u002B 1] * weight.Data[weightRow \u002B 1];\n\t\tfloat product2 = input.Data[inputRow \u002B 2] * weight.Data[weightRow \u002B 2];\n\t\tfloat product3 = input.Data[inputRow \u002B 3] * weight.Data[weightRow \u002B 3];\n\t\tfor ( int inputFeature = 0; inputFeature \u003C inputSize; inputFeature\u002B\u002B )\n\t\t{\n\t\t\tsum = MathF.FusedMultiplyAdd(\n\t\t\t\tinput.Data[inputRow \u002B inputFeature],\n\t\t\t\tweight.Data[weightRow \u002B inputFeature],\n\t\t\t\tsum );\n\t\t}\n\n\t\tfloat biasValue = bias is null ? 0 : bias.Data[outputFeature];\n\t\tfloat recomputed = bias is null ? sum : sum \u002B biasValue;\n\t\tstring formula = bias is null\n\t\t\t? \u0022sum(input[token,i]*weight[out,i])\u0022\n\t\t\t: \u0022sum(input[token,i]*weight[out,i])\u002Bbias[out]\u0022;\n\t\treturn $\u0022token={token} output_feature={outputFeature} input_length={inputSize} \u0022 \u002B\n\t\t\t$\u0022formula={formula} expected={expected:G9} actual={actual:G9} \u0022 \u002B\n\t\t\t$\u0022dot={sum:G9} bias={biasValue:G9} recomputed={recomputed:G9} \u0022 \u002B\n\t\t\t$\u0022first_products=[{product0:G9},{product1:G9},{product2:G9},{product3:G9}]\u0022;\n\t}\n\n\tpublic static Tensor ApplyGeluNew( Tensor input, string outputName )\n\t{\n\t\tif ( input is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( input ) );\n\t\t}\n\t\tif ( string.IsNullOrWhiteSpace( outputName ) )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t\u0022[LLM:ERROR] GELU-new output name cannot be empty.\u0022,\n\t\t\t\tnameof( outputName ) );\n\t\t}\n\t\tif ( input.Rank != 2 || input.Shape[0] \u003C= 0 || input.Shape[1] \u003C= 0 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {outputName} GELU-new input expected a positive rank-2 \u0022 \u002B\n\t\t\t\t$\u0022[sequence,feature] tensor, found {input.ShapeText}.\u0022 );\n\t\t}\n\n\t\tfloat[] output = new float[input.Data.Length];\n\t\tfor ( int index = 0; index \u003C input.Data.Length; index\u002B\u002B )\n\t\t{\n\t\t\tfloat value = input.Data[index];\n\t\t\tif ( !float.IsFinite( value ) )\n\t\t\t{\n\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t$\u0022[LLM:ERROR] {outputName} GELU-new input contains non-finite \u0022 \u002B\n\t\t\t\t\t$\u0022value at flat index {index}: {value}.\u0022 );\n\t\t\t}\n\t\t\tfloat cube = MathF.Pow( value, 3.0f );\n\t\t\tfloat inner = value \u002B GeluNewCubicCoefficient * cube;\n\t\t\tfloat tanhArgument = GeluNewTanhCoefficient * inner;\n\t\t\tfloat tanhValue = MathF.Tanh( tanhArgument );\n\t\t\tfloat activated = (0.5f * value) * (1.0f \u002B tanhValue);\n\t\t\tif ( !float.IsFinite( activated ) )\n\t\t\t{\n\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t$\u0022[LLM:ERROR] {outputName} GELU-new produced non-finite output \u0022 \u002B\n\t\t\t\t\t$\u0022at flat index {index}: input={value:G9}, cube={cube:G9}, \u0022 \u002B\n\t\t\t\t\t$\u0022tanh_argument={tanhArgument:G9}, output={activated}.\u0022 );\n\t\t\t}\n\t\t\toutput[index] = activated;\n\t\t}\n\n\t\treturn new Tensor(\n\t\t\toutputName,\n\t\t\tnew[] { input.Shape[0], input.Shape[1] },\n\t\t\toutput );\n\t}\n\n\tpublic static string DescribeGeluNewElement(\n\t\tTensor input,\n\t\tTensor output,\n\t\tint token,\n\t\tint feature,\n\t\tfloat expected )\n\t{\n\t\tif ( input is null || output is null || input.Rank != 2 || output.Rank != 2 ||\n\t\t\tinput.Shape[0] != output.Shape[0] || input.Shape[1] != output.Shape[1] )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\u0022[LLM:ERROR] GELU-new diagnostic requires matching rank-2 tensors.\u0022 );\n\t\t}\n\t\tif ( token \u003C 0 || token \u003E= input.Shape[0] || feature \u003C 0 || feature \u003E= input.Shape[1] )\n\t\t{\n\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t$\u0022[LLM:ERROR] GELU-new diagnostic index [{token},{feature}] is \u0022 \u002B\n\t\t\t\t$\u0022outside {input.ShapeText}.\u0022 );\n\t\t}\n\n\t\tint index = token * input.Shape[1] \u002B feature;\n\t\tfloat value = input.Data[index];\n\t\tfloat cube = MathF.Pow( value, 3.0f );\n\t\tfloat inner = value \u002B GeluNewCubicCoefficient * cube;\n\t\tfloat tanhArgument = GeluNewTanhCoefficient * inner;\n\t\tfloat tanhValue = MathF.Tanh( tanhArgument );\n\t\tfloat recomputed = (0.5f * value) * (1.0f \u002B tanhValue);\n\t\tfloat actual = output.Data[index];\n\t\treturn $\u0022token={token} feature={feature} input={value:G9} cube={cube:G9} \u0022 \u002B\n\t\t\t$\u0022cubic_coefficient={GeluNewCubicCoefficient:G9} inner={inner:G9} \u0022 \u002B\n\t\t\t$\u0022tanh_coefficient={GeluNewTanhCoefficient:G9} \u0022 \u002B\n\t\t\t$\u0022tanh_argument={tanhArgument:G9} tanh={tanhValue:G9} \u0022 \u002B\n\t\t\t$\u0022recomputed={recomputed:G9} actual={actual:G9} python={expected:G9} \u0022 \u002B\n\t\t\t$\u0022abs_diff={MathF.Abs( actual - expected ):G12}\u0022;\n\t}\n\n\tpublic static Tensor AddMlpResidual(\n\t\tTensor residualSource,\n\t\tTensor mlpBranch,\n\t\tstring outputName )\n\t{\n\t\tif ( residualSource is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( residualSource ) );\n\t\t}\n\t\tif ( mlpBranch is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( mlpBranch ) );\n\t\t}\n\t\tif ( string.IsNullOrWhiteSpace( outputName ) )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t\u0022[LLM:ERROR] MLP residual output name cannot be empty.\u0022,\n\t\t\t\tnameof( outputName ) );\n\t\t}\n\t\tif ( residualSource.Rank != 2 || mlpBranch.Rank != 2 ||\n\t\t\tresidualSource.Shape[0] != mlpBranch.Shape[0] ||\n\t\t\tresidualSource.Shape[1] != mlpBranch.Shape[1] ||\n\t\t\tresidualSource.Data.Length != mlpBranch.Data.Length )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {outputName} MLP residual addition requires matching \u0022 \u002B\n\t\t\t\t$\u0022rank-2 tensors, found residual={residualSource.ShapeText}, \u0022 \u002B\n\t\t\t\t$\u0022MLP={mlpBranch.ShapeText}.\u0022 );\n\t\t}\n\n\t\tfloat[] output = new float[residualSource.Data.Length];\n\t\tfor ( int index = 0; index \u003C output.Length; index\u002B\u002B )\n\t\t{\n\t\t\t// Match GPTNeoBlock.forward: residual \u002B feed_forward_hidden_states.\n\t\t\toutput[index] = residualSource.Data[index] \u002B mlpBranch.Data[index];\n\t\t}\n\t\treturn new Tensor(\n\t\t\toutputName,\n\t\t\tnew[] { residualSource.Shape[0], residualSource.Shape[1] },\n\t\t\toutput );\n\t}\n\n\tpublic static string DescribeMlpResidualAddition(\n\t\tTensor residualSource,\n\t\tTensor mlpBranch,\n\t\tTensor output,\n\t\tint token,\n\t\tint feature,\n\t\tfloat expected )\n\t{\n\t\tif ( residualSource is null || mlpBranch is null || output is null ||\n\t\t\tresidualSource.Rank != 2 || mlpBranch.Rank != 2 || output.Rank != 2 ||\n\t\t\tresidualSource.Shape[0] != mlpBranch.Shape[0] ||\n\t\t\tresidualSource.Shape[1] != mlpBranch.Shape[1] ||\n\t\t\tresidualSource.Shape[0] != output.Shape[0] ||\n\t\t\tresidualSource.Shape[1] != output.Shape[1] )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\u0022[LLM:ERROR] MLP residual diagnostic requires matching rank-2 tensors.\u0022 );\n\t\t}\n\t\tif ( token \u003C 0 || token \u003E= output.Shape[0] || feature \u003C 0 || feature \u003E= output.Shape[1] )\n\t\t{\n\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t$\u0022[LLM:ERROR] MLP residual diagnostic index [{token},{feature}] is \u0022 \u002B\n\t\t\t\t$\u0022outside {output.ShapeText}.\u0022 );\n\t\t}\n\n\t\tint index = token * output.Shape[1] \u002B feature;\n\t\tfloat residual = residualSource.Data[index];\n\t\tfloat branch = mlpBranch.Data[index];\n\t\tfloat actual = output.Data[index];\n\t\treturn $\u0022token={token} feature={feature} residual_source={residual:G9} \u0022 \u002B\n\t\t\t$\u0022mlp_branch={branch:G9} sum={actual:G9} python={expected:G9} \u0022 \u002B\n\t\t\t$\u0022abs_diff={MathF.Abs( expected - actual ):G12}\u0022;\n\t}\n\n\tpublic static Tensor AddResidual(\n\t\tTensor attentionBranch,\n\t\tTensor residualSource,\n\t\tstring outputName )\n\t{\n\t\tif ( attentionBranch is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( attentionBranch ) );\n\t\t}\n\t\tif ( residualSource is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( residualSource ) );\n\t\t}\n\t\tif ( string.IsNullOrWhiteSpace( outputName ) )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t\u0022[LLM:ERROR] Residual-add output name cannot be empty.\u0022,\n\t\t\t\tnameof( outputName ) );\n\t\t}\n\t\tif ( attentionBranch.Rank != 2 || residualSource.Rank != 2 ||\n\t\t\tattentionBranch.Shape[0] != residualSource.Shape[0] ||\n\t\t\tattentionBranch.Shape[1] != residualSource.Shape[1] ||\n\t\t\tattentionBranch.Data.Length != residualSource.Data.Length )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {outputName} residual addition requires matching rank-2 \u0022 \u002B\n\t\t\t\t$\u0022tensors, found attention={attentionBranch.ShapeText}, \u0022 \u002B\n\t\t\t\t$\u0022residual={residualSource.ShapeText}.\u0022 );\n\t\t}\n\n\t\tfloat[] output = new float[attentionBranch.Data.Length];\n\t\tfor ( int index = 0; index \u003C output.Length; index\u002B\u002B )\n\t\t{\n\t\t\toutput[index] = attentionBranch.Data[index] \u002B residualSource.Data[index];\n\t\t}\n\t\treturn new Tensor(\n\t\t\toutputName,\n\t\t\tnew[] { attentionBranch.Shape[0], attentionBranch.Shape[1] },\n\t\t\toutput );\n\t}\n\n\tpublic static string DescribeResidualAddition(\n\t\tTensor attentionBranch,\n\t\tTensor residualSource,\n\t\tTensor output,\n\t\tint token,\n\t\tint feature,\n\t\tfloat expected )\n\t{\n\t\tif ( attentionBranch is null || residualSource is null || output is null ||\n\t\t\tattentionBranch.Rank != 2 || residualSource.Rank != 2 || output.Rank != 2 ||\n\t\t\tattentionBranch.Shape[0] != residualSource.Shape[0] ||\n\t\t\tattentionBranch.Shape[1] != residualSource.Shape[1] ||\n\t\t\tattentionBranch.Shape[0] != output.Shape[0] ||\n\t\t\tattentionBranch.Shape[1] != output.Shape[1] )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\u0022[LLM:ERROR] Residual diagnostic requires compatible rank-2 attention, \u0022 \u002B\n\t\t\t\t\u0022residual, and output tensors.\u0022 );\n\t\t}\n\t\tif ( token \u003C 0 || token \u003E= output.Shape[0] ||\n\t\t\tfeature \u003C 0 || feature \u003E= output.Shape[1] )\n\t\t{\n\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t$\u0022[LLM:ERROR] Residual diagnostic index [{token},{feature}] is \u0022 \u002B\n\t\t\t\t$\u0022outside output shape {output.ShapeText}.\u0022 );\n\t\t}\n\n\t\tint index = token * output.Shape[1] \u002B feature;\n\t\tfloat branch = attentionBranch.Data[index];\n\t\tfloat residual = residualSource.Data[index];\n\t\tfloat actual = output.Data[index];\n\t\treturn $\u0022token={token} feature={feature} residual_source={residual:G9} \u0022 \u002B\n\t\t\t$\u0022attention_branch={branch:G9} sum={actual:G9} python={expected:G9} \u0022 \u002B\n\t\t\t$\u0022abs_diff={MathF.Abs( expected - actual ):G12}\u0022;\n\t}\n\n\tpublic static Tensor SplitHeads( Tensor projection, int headCount, string outputName )\n\t{\n\t\tif ( projection is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( projection ) );\n\t\t}\n\t\tif ( string.IsNullOrWhiteSpace( outputName ) )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t\u0022[LLM:ERROR] Head-split output name cannot be empty.\u0022,\n\t\t\t\tnameof( outputName ) );\n\t\t}\n\t\tif ( projection.Rank != 2 || projection.Shape[0] \u003C= 0 || projection.Shape[1] \u003C= 0 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {outputName} head split expected projection shape \u0022 \u002B\n\t\t\t\t$\u0022[sequence,hidden] with positive dimensions, found {projection.ShapeText}.\u0022 );\n\t\t}\n\t\tif ( headCount \u003C= 0 || projection.Shape[1] % headCount != 0 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {outputName} hidden size {projection.Shape[1]} must be divisible \u0022 \u002B\n\t\t\t\t$\u0022by positive head count {headCount}.\u0022 );\n\t\t}\n\n\t\tint sequenceLength = projection.Shape[0];\n\t\tint hiddenSize = projection.Shape[1];\n\t\tint headDimension = hiddenSize / headCount;\n\t\tfloat[] output = new float[projection.Data.Length];\n\t\tfor ( int head = 0; head \u003C headCount; head\u002B\u002B )\n\t\t{\n\t\t\tfor ( int token = 0; token \u003C sequenceLength; token\u002B\u002B )\n\t\t\t{\n\t\t\t\tint sourceRow = token * hiddenSize;\n\t\t\t\tint destinationRow = (head * sequenceLength \u002B token) * headDimension;\n\t\t\t\tfor ( int component = 0; component \u003C headDimension; component\u002B\u002B )\n\t\t\t\t{\n\t\t\t\t\tint sourceFeature = head * headDimension \u002B component;\n\t\t\t\t\toutput[destinationRow \u002B component] = projection.Data[sourceRow \u002B sourceFeature];\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn new Tensor(\n\t\t\toutputName,\n\t\t\tnew[] { headCount, sequenceLength, headDimension },\n\t\t\toutput );\n\t}\n\n\tpublic static string DescribeHeadMapping(\n\t\tTensor projection,\n\t\tTensor heads,\n\t\tint head,\n\t\tint token,\n\t\tint component )\n\t{\n\t\tif ( projection is null || heads is null || projection.Rank != 2 || heads.Rank != 3 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\u0022[LLM:ERROR] Head mapping diagnostic requires rank-2 projection and rank-3 heads.\u0022 );\n\t\t}\n\t\tint headCount = heads.Shape[0];\n\t\tint sequenceLength = heads.Shape[1];\n\t\tint headDimension = heads.Shape[2];\n\t\tif ( projection.Shape[0] != sequenceLength ||\n\t\t\tprojection.Shape[1] != headCount * headDimension )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] Head mapping diagnostic incompatible shapes: \u0022 \u002B\n\t\t\t\t$\u0022projection={projection.ShapeText}, heads={heads.ShapeText}.\u0022 );\n\t\t}\n\t\tif ( head \u003C 0 || head \u003E= headCount || token \u003C 0 || token \u003E= sequenceLength ||\n\t\t\tcomponent \u003C 0 || component \u003E= headDimension )\n\t\t{\n\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t$\u0022[LLM:ERROR] Head mapping diagnostic index [{head},{token},{component}] \u0022 \u002B\n\t\t\t\t$\u0022is outside {heads.ShapeText}.\u0022 );\n\t\t}\n\n\t\tint sourceFeature = head * headDimension \u002B component;\n\t\tint sourceIndex = token * projection.Shape[1] \u002B sourceFeature;\n\t\tint destinationIndex = (head * sequenceLength \u002B token) * headDimension \u002B component;\n\t\tfloat source = projection.Data[sourceIndex];\n\t\tfloat destination = heads.Data[destinationIndex];\n\t\treturn $\u0022head={head} token={token} component={component} source_feature={sourceFeature} \u0022 \u002B\n\t\t\t$\u0022source_index={sourceIndex} destination_index={destinationIndex} \u0022 \u002B\n\t\t\t$\u0022source={source:G9} destination={destination:G9} \u0022 \u002B\n\t\t\t$\u0022source_bits={BitConverter.SingleToInt32Bits( source ):X8} \u0022 \u002B\n\t\t\t$\u0022destination_bits={BitConverter.SingleToInt32Bits( destination ):X8}\u0022;\n\t}\n\n\tpublic static Tensor MergeHeads( Tensor contextHeads, string outputName )\n\t{\n\t\tif ( contextHeads is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( contextHeads ) );\n\t\t}\n\t\tif ( string.IsNullOrWhiteSpace( outputName ) )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t\u0022[LLM:ERROR] Head-merge output name cannot be empty.\u0022,\n\t\t\t\tnameof( outputName ) );\n\t\t}\n\t\tif ( contextHeads.Rank != 3 || contextHeads.Shape[0] \u003C= 0 ||\n\t\t\tcontextHeads.Shape[1] \u003C= 0 || contextHeads.Shape[2] \u003C= 0 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {outputName} head merge expected positive shape \u0022 \u002B\n\t\t\t\t$\u0022[head,sequence,head_dimension], found {contextHeads.ShapeText}.\u0022 );\n\t\t}\n\n\t\tint headCount = contextHeads.Shape[0];\n\t\tint sequenceLength = contextHeads.Shape[1];\n\t\tint headDimension = contextHeads.Shape[2];\n\t\tif ( headCount \u003E int.MaxValue / headDimension )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {outputName} hidden size {headCount}*{headDimension} \u0022 \u002B\n\t\t\t\t\u0022exceeds the managed array limit.\u0022 );\n\t\t}\n\t\tint hiddenSize = headCount * headDimension;\n\t\tif ( sequenceLength \u003E int.MaxValue / hiddenSize )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {outputName} output shape [{sequenceLength},{hiddenSize}] \u0022 \u002B\n\t\t\t\t\u0022exceeds the managed array limit.\u0022 );\n\t\t}\n\n\t\tfloat[] output = new float[sequenceLength * hiddenSize];\n\t\tfor ( int token = 0; token \u003C sequenceLength; token\u002B\u002B )\n\t\t{\n\t\t\tint destinationRow = token * hiddenSize;\n\t\t\tfor ( int head = 0; head \u003C headCount; head\u002B\u002B )\n\t\t\t{\n\t\t\t\tint sourceRow = (head * sequenceLength \u002B token) * headDimension;\n\t\t\t\tint destinationHead = destinationRow \u002B head * headDimension;\n\t\t\t\tfor ( int component = 0; component \u003C headDimension; component\u002B\u002B )\n\t\t\t\t{\n\t\t\t\t\toutput[destinationHead \u002B component] =\n\t\t\t\t\t\tcontextHeads.Data[sourceRow \u002B component];\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn new Tensor(\n\t\t\toutputName,\n\t\t\tnew[] { sequenceLength, hiddenSize },\n\t\t\toutput );\n\t}\n\n\tpublic static string DescribeHeadMergeMapping(\n\t\tTensor contextHeads,\n\t\tTensor merged,\n\t\tint token,\n\t\tint feature )\n\t{\n\t\tif ( contextHeads is null || merged is null ||\n\t\t\tcontextHeads.Rank != 3 || merged.Rank != 2 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\u0022[LLM:ERROR] Head-merge diagnostic requires rank-3 context heads and \u0022 \u002B\n\t\t\t\t\u0022rank-2 merged context.\u0022 );\n\t\t}\n\t\tint headCount = contextHeads.Shape[0];\n\t\tint sequenceLength = contextHeads.Shape[1];\n\t\tint headDimension = contextHeads.Shape[2];\n\t\tint hiddenSize = headCount * headDimension;\n\t\tif ( merged.Shape[0] != sequenceLength || merged.Shape[1] != hiddenSize )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] Head-merge diagnostic incompatible shapes: \u0022 \u002B\n\t\t\t\t$\u0022context={contextHeads.ShapeText}, merged={merged.ShapeText}.\u0022 );\n\t\t}\n\t\tif ( token \u003C 0 || token \u003E= sequenceLength || feature \u003C 0 || feature \u003E= hiddenSize )\n\t\t{\n\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t$\u0022[LLM:ERROR] Head-merge diagnostic index token={token}, feature={feature} \u0022 \u002B\n\t\t\t\t$\u0022is outside merged shape {merged.ShapeText}.\u0022 );\n\t\t}\n\n\t\tint head = feature / headDimension;\n\t\tint component = feature % headDimension;\n\t\tint sourceIndex = (head * sequenceLength \u002B token) * headDimension \u002B component;\n\t\tint destinationIndex = token * hiddenSize \u002B feature;\n\t\tfloat source = contextHeads.Data[sourceIndex];\n\t\tfloat destination = merged.Data[destinationIndex];\n\t\treturn $\u0022token={token} feature={feature} head={head} component={component} \u0022 \u002B\n\t\t\t$\u0022source_index={sourceIndex} destination_index={destinationIndex} \u0022 \u002B\n\t\t\t$\u0022source={source:G9} merged={destination:G9} \u0022 \u002B\n\t\t\t$\u0022source_bits={BitConverter.SingleToInt32Bits( source ):X8} \u0022 \u002B\n\t\t\t$\u0022merged_bits={BitConverter.SingleToInt32Bits( destination ):X8}\u0022;\n\t}\n\n\tpublic static Tensor ComputeScaledUnmaskedAttentionScores(\n\t\tTensor queryHeads,\n\t\tTensor keyHeads,\n\t\tfloat scale,\n\t\tstring outputName )\n\t{\n\t\tif ( queryHeads is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( queryHeads ) );\n\t\t}\n\t\tif ( keyHeads is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( keyHeads ) );\n\t\t}\n\t\tif ( string.IsNullOrWhiteSpace( outputName ) )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t\u0022[LLM:ERROR] Attention-score output name cannot be empty.\u0022,\n\t\t\t\tnameof( outputName ) );\n\t\t}\n\t\tif ( queryHeads.Rank != 3 || keyHeads.Rank != 3 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {outputName} expected Q/K heads rank 3 \u0022 \u002B\n\t\t\t\t$\u0022[head,sequence,component], found Q={queryHeads.ShapeText}, \u0022 \u002B\n\t\t\t\t$\u0022K={keyHeads.ShapeText}.\u0022 );\n\t\t}\n\t\tif ( queryHeads.Shape[0] != keyHeads.Shape[0] ||\n\t\t\tqueryHeads.Shape[2] != keyHeads.Shape[2] )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {outputName} Q/K head count and component dimensions must match; \u0022 \u002B\n\t\t\t\t$\u0022found Q={queryHeads.ShapeText}, K={keyHeads.ShapeText}.\u0022 );\n\t\t}\n\t\tif ( !float.IsFinite( scale ) || !(scale \u003E 0) )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {outputName} attention scale must be positive and finite, \u0022 \u002B\n\t\t\t\t$\u0022found {scale:G9}.\u0022 );\n\t\t}\n\n\t\tint headCount = queryHeads.Shape[0];\n\t\tint queryLength = queryHeads.Shape[1];\n\t\tint keyLength = keyHeads.Shape[1];\n\t\tint headDimension = queryHeads.Shape[2];\n\t\tif ( headCount \u003E int.MaxValue / queryLength ||\n\t\t\theadCount * queryLength \u003E int.MaxValue / keyLength )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {outputName} shape [{headCount},{queryLength},{keyLength}] \u0022 \u002B\n\t\t\t\t\u0022exceeds the managed array limit.\u0022 );\n\t\t}\n\n\t\tfloat[] output = new float[headCount * queryLength * keyLength];\n\t\tfor ( int head = 0; head \u003C headCount; head\u002B\u002B )\n\t\t{\n\t\t\tfor ( int query = 0; query \u003C queryLength; query\u002B\u002B )\n\t\t\t{\n\t\t\t\tint queryRow = (head * queryLength \u002B query) * headDimension;\n\t\t\t\tfor ( int key = 0; key \u003C keyLength; key\u002B\u002B )\n\t\t\t\t{\n\t\t\t\t\tint keyRow = (head * keyLength \u002B key) * headDimension;\n\t\t\t\t\tfloat rawSum = 0;\n\t\t\t\t\tfor ( int component = 0; component \u003C headDimension; component\u002B\u002B )\n\t\t\t\t\t{\n\t\t\t\t\t\trawSum \u002B= queryHeads.Data[queryRow \u002B component] *\n\t\t\t\t\t\t\tkeyHeads.Data[keyRow \u002B component];\n\t\t\t\t\t}\n\n\t\t\t\t\t// Transformers 5.15 GPT-Neo performs no inverse-sqrt scaling.\n\t\t\t\t\t// Avoid adding an operation when the exact model-equivalent factor is 1.\n\t\t\t\t\toutput[(head * queryLength \u002B query) * keyLength \u002B key] =\n\t\t\t\t\t\tscale == 1.0f ? rawSum : rawSum * scale;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn new Tensor(\n\t\t\toutputName,\n\t\t\tnew[] { headCount, queryLength, keyLength },\n\t\t\toutput );\n\t}\n\n\tpublic static bool IsLayer0AttentionAllowed(\n\t\tstring attentionType,\n\t\tint query,\n\t\tint key,\n\t\tint queryLength,\n\t\tint keyLength )\n\t{\n\t\treturn IsAttentionAllowed(\n\t\t\tattentionType,\n\t\t\twindowSize: 0,\n\t\t\tquery,\n\t\t\tkey,\n\t\t\tqueryLength,\n\t\t\tkeyLength );\n\t}\n\n\tpublic static bool IsAttentionAllowed(\n\t\tstring attentionType,\n\t\tint windowSize,\n\t\tint query,\n\t\tint key,\n\t\tint queryLength,\n\t\tint keyLength )\n\t{\n\t\tif ( attentionType != \u0022global\u0022 \u0026\u0026 attentionType != \u0022local\u0022 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] GPT-Neo attention type must be global or local, \u0022 \u002B\n\t\t\t\t$\u0022found \u0027{attentionType}\u0027.\u0022 );\n\t\t}\n\t\tif ( attentionType == \u0022local\u0022 \u0026\u0026 windowSize \u003C= 0 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] Local GPT-Neo attention window must be positive, \u0022 \u002B\n\t\t\t\t$\u0022found {windowSize}.\u0022 );\n\t\t}\n\t\tif ( queryLength \u003C= 0 || keyLength \u003C= 0 || queryLength \u003E keyLength ||\n\t\t\tquery \u003C 0 || query \u003E= queryLength || key \u003C 0 || key \u003E= keyLength )\n\t\t{\n\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t$\u0022[LLM:ERROR] Attention mask index query={query}, key={key} is invalid \u0022 \u002B\n\t\t\t\t$\u0022for query_length={queryLength}, key_length={keyLength}. \u0022 \u002B\n\t\t\t\t\u0022The verified source slice requires 0 \u003C query_length \u003C= key_length.\u0022 );\n\t\t}\n\n\t\tint absoluteQuery = keyLength - queryLength \u002B query;\n\t\tbool causal = key \u003C= absoluteQuery;\n\t\treturn attentionType == \u0022global\u0022\n\t\t\t? causal\n\t\t\t: causal \u0026\u0026 absoluteQuery - key \u003C windowSize;\n\t}\n\n\tpublic static Tensor ApplyLayer0AttentionMask(\n\t\tTensor unmaskedScores,\n\t\tstring attentionType,\n\t\tfloat maskedSentinel,\n\t\tstring outputName )\n\t{\n\t\treturn ApplyAttentionMask(\n\t\t\tunmaskedScores,\n\t\t\tattentionType,\n\t\t\twindowSize: 0,\n\t\t\tmaskedSentinel,\n\t\t\toutputName );\n\t}\n\n\tpublic static Tensor ApplyAttentionMask(\n\t\tTensor unmaskedScores,\n\t\tstring attentionType,\n\t\tint windowSize,\n\t\tfloat maskedSentinel,\n\t\tstring outputName )\n\t{\n\t\tif ( unmaskedScores is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( unmaskedScores ) );\n\t\t}\n\t\tif ( string.IsNullOrWhiteSpace( outputName ) )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t\u0022[LLM:ERROR] Masked attention-score output name cannot be empty.\u0022,\n\t\t\t\tnameof( outputName ) );\n\t\t}\n\t\tif ( unmaskedScores.Rank != 3 || unmaskedScores.Shape[0] \u003C= 0 ||\n\t\t\tunmaskedScores.Shape[1] \u003C= 0 || unmaskedScores.Shape[2] \u003C= 0 ||\n\t\t\tunmaskedScores.Shape[1] \u003E unmaskedScores.Shape[2] )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {outputName} expected unmasked scores shape \u0022 \u002B\n\t\t\t\t$\u0022[head,query,key] with 0 \u003C query \u003C= key, found \u0022 \u002B\n\t\t\t\t$\u0022{unmaskedScores.ShapeText}.\u0022 );\n\t\t}\n\t\tif ( !float.IsFinite( maskedSentinel ) || !(maskedSentinel \u003C 0) )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {outputName} mask sentinel must be finite and negative, \u0022 \u002B\n\t\t\t\t$\u0022found {maskedSentinel:G9}.\u0022 );\n\t\t}\n\n\t\tint headCount = unmaskedScores.Shape[0];\n\t\tint queryLength = unmaskedScores.Shape[1];\n\t\tint keyLength = unmaskedScores.Shape[2];\n\t\tfloat[] output = new float[unmaskedScores.Data.Length];\n\t\tfor ( int head = 0; head \u003C headCount; head\u002B\u002B )\n\t\t{\n\t\t\tfor ( int query = 0; query \u003C queryLength; query\u002B\u002B )\n\t\t\t{\n\t\t\t\tfor ( int key = 0; key \u003C keyLength; key\u002B\u002B )\n\t\t\t\t{\n\t\t\t\t\tint index = (head * queryLength \u002B query) * keyLength \u002B key;\n\t\t\t\t\tbool allowed = IsAttentionAllowed(\n\t\t\t\t\t\tattentionType,\n\t\t\t\t\t\twindowSize,\n\t\t\t\t\t\tquery,\n\t\t\t\t\t\tkey,\n\t\t\t\t\t\tqueryLength,\n\t\t\t\t\t\tkeyLength );\n\t\t\t\t\toutput[index] = allowed ? unmaskedScores.Data[index] : maskedSentinel;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn new Tensor(\n\t\t\toutputName,\n\t\t\tnew[] { headCount, queryLength, keyLength },\n\t\t\toutput );\n\t}\n\n\tpublic static Tensor ComputeAttentionProbabilities(\n\t\tTensor maskedScores,\n\t\tstring outputName )\n\t{\n\t\tif ( maskedScores is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( maskedScores ) );\n\t\t}\n\t\tif ( string.IsNullOrWhiteSpace( outputName ) )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t\u0022[LLM:ERROR] Attention-probability output name cannot be empty.\u0022,\n\t\t\t\tnameof( outputName ) );\n\t\t}\n\t\tif ( maskedScores.Rank != 3 || maskedScores.Shape[0] \u003C= 0 ||\n\t\t\tmaskedScores.Shape[1] \u003C= 0 || maskedScores.Shape[2] \u003C= 0 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {outputName} expected masked scores shape \u0022 \u002B\n\t\t\t\t$\u0022[head,query,key] with positive dimensions, found \u0022 \u002B\n\t\t\t\t$\u0022{maskedScores.ShapeText}.\u0022 );\n\t\t}\n\n\t\tint headCount = maskedScores.Shape[0];\n\t\tint queryLength = maskedScores.Shape[1];\n\t\tint keyLength = maskedScores.Shape[2];\n\t\tfloat[] output = new float[maskedScores.Data.Length];\n\t\tfor ( int head = 0; head \u003C headCount; head\u002B\u002B )\n\t\t{\n\t\t\tfor ( int query = 0; query \u003C queryLength; query\u002B\u002B )\n\t\t\t{\n\t\t\t\tint row = (head * queryLength \u002B query) * keyLength;\n\t\t\t\tfloat rowMaximum = float.NegativeInfinity;\n\t\t\t\tfor ( int key = 0; key \u003C keyLength; key\u002B\u002B )\n\t\t\t\t{\n\t\t\t\t\tfloat value = maskedScores.Data[row \u002B key];\n\t\t\t\t\tif ( !float.IsFinite( value ) )\n\t\t\t\t\t{\n\t\t\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t\t\t$\u0022[LLM:ERROR] {outputName} input contains a non-finite score \u0022 \u002B\n\t\t\t\t\t\t\t$\u0022at [{head},{query},{key}]: {value}.\u0022 );\n\t\t\t\t\t}\n\t\t\t\t\trowMaximum = MathF.Max( rowMaximum, value );\n\t\t\t\t}\n\n\t\t\t\tfloat exponentialSum = 0;\n\t\t\t\tfor ( int key = 0; key \u003C keyLength; key\u002B\u002B )\n\t\t\t\t{\n\t\t\t\t\tfloat shifted = maskedScores.Data[row \u002B key] - rowMaximum;\n\t\t\t\t\tfloat exponential = MathF.Exp( shifted );\n\t\t\t\t\tif ( !float.IsFinite( exponential ) || exponential \u003C 0 )\n\t\t\t\t\t{\n\t\t\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t\t\t$\u0022[LLM:ERROR] {outputName} produced invalid exp at \u0022 \u002B\n\t\t\t\t\t\t\t$\u0022[{head},{query},{key}]: input={maskedScores.Data[row \u002B key]:G9}, \u0022 \u002B\n\t\t\t\t\t\t\t$\u0022maximum={rowMaximum:G9}, shifted={shifted:G9}, \u0022 \u002B\n\t\t\t\t\t\t\t$\u0022exp={exponential:G9}.\u0022 );\n\t\t\t\t\t}\n\t\t\t\t\toutput[row \u002B key] = exponential;\n\t\t\t\t\texponentialSum \u002B= exponential;\n\t\t\t\t}\n\n\t\t\t\tif ( !(exponentialSum \u003E 0) || !float.IsFinite( exponentialSum ) )\n\t\t\t\t{\n\t\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t\t$\u0022[LLM:ERROR] {outputName} row [{head},{query}] produced invalid \u0022 \u002B\n\t\t\t\t\t\t$\u0022exponential sum {exponentialSum:G9}.\u0022 );\n\t\t\t\t}\n\n\t\t\t\tfloat inverseSum = 1.0f / exponentialSum;\n\t\t\t\tfor ( int key = 0; key \u003C keyLength; key\u002B\u002B )\n\t\t\t\t{\n\t\t\t\t\toutput[row \u002B key] *= inverseSum;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn new Tensor(\n\t\t\toutputName,\n\t\t\tnew[] { headCount, queryLength, keyLength },\n\t\t\toutput );\n\t}\n\n\tpublic static Tensor ComputeAttentionContextHeads(\n\t\tTensor probabilities,\n\t\tTensor valueHeads,\n\t\tstring outputName )\n\t{\n\t\tif ( probabilities is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( probabilities ) );\n\t\t}\n\t\tif ( valueHeads is null )\n\t\t{\n\t\t\tthrow new ArgumentNullException( nameof( valueHeads ) );\n\t\t}\n\t\tif ( string.IsNullOrWhiteSpace( outputName ) )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t\u0022[LLM:ERROR] Attention-context output name cannot be empty.\u0022,\n\t\t\t\tnameof( outputName ) );\n\t\t}\n\t\tif ( probabilities.Rank != 3 || probabilities.Shape[0] \u003C= 0 ||\n\t\t\tprobabilities.Shape[1] \u003C= 0 || probabilities.Shape[2] \u003C= 0 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {outputName} expected probabilities shape \u0022 \u002B\n\t\t\t\t$\u0022[head,query,key] with positive dimensions, found \u0022 \u002B\n\t\t\t\t$\u0022{probabilities.ShapeText}.\u0022 );\n\t\t}\n\t\tif ( valueHeads.Rank != 3 || valueHeads.Shape[0] \u003C= 0 ||\n\t\t\tvalueHeads.Shape[1] \u003C= 0 || valueHeads.Shape[2] \u003C= 0 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {outputName} expected value shape \u0022 \u002B\n\t\t\t\t$\u0022[head,key,component] with positive dimensions, found \u0022 \u002B\n\t\t\t\t$\u0022{valueHeads.ShapeText}.\u0022 );\n\t\t}\n\t\tif ( probabilities.Shape[0] != valueHeads.Shape[0] ||\n\t\t\tprobabilities.Shape[2] != valueHeads.Shape[1] )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {outputName} incompatible probabilities/value shapes: \u0022 \u002B\n\t\t\t\t$\u0022probabilities={probabilities.ShapeText} [head,query,key], \u0022 \u002B\n\t\t\t\t$\u0022V={valueHeads.ShapeText} [head,key,component].\u0022 );\n\t\t}\n\n\t\tint headCount = probabilities.Shape[0];\n\t\tint queryLength = probabilities.Shape[1];\n\t\tint keyLength = probabilities.Shape[2];\n\t\tint headDimension = valueHeads.Shape[2];\n\t\tfloat[] output = new float[headCount * queryLength * headDimension];\n\t\tfor ( int head = 0; head \u003C headCount; head\u002B\u002B )\n\t\t{\n\t\t\tfor ( int query = 0; query \u003C queryLength; query\u002B\u002B )\n\t\t\t{\n\t\t\t\tint probabilityRow = (head * queryLength \u002B query) * keyLength;\n\t\t\t\tfor ( int component = 0; component \u003C headDimension; component\u002B\u002B )\n\t\t\t\t{\n\t\t\t\t\tfloat sum = 0;\n\t\t\t\t\tfor ( int key = 0; key \u003C keyLength; key\u002B\u002B )\n\t\t\t\t\t{\n\t\t\t\t\t\tfloat probability = probabilities.Data[probabilityRow \u002B key];\n\t\t\t\t\t\tint valueIndex = (head * keyLength \u002B key) * headDimension \u002B component;\n\t\t\t\t\t\tfloat value = valueHeads.Data[valueIndex];\n\t\t\t\t\t\tif ( !float.IsFinite( probability ) || !float.IsFinite( value ) )\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t\t\t\t$\u0022[LLM:ERROR] {outputName} received non-finite input at \u0022 \u002B\n\t\t\t\t\t\t\t\t$\u0022head={head}, query={query}, key={key}, component={component}: \u0022 \u002B\n\t\t\t\t\t\t\t\t$\u0022probability={probability:G9}, V={value:G9}.\u0022 );\n\t\t\t\t\t\t}\n\t\t\t\t\t\tsum \u002B= probability * value;\n\t\t\t\t\t}\n\n\t\t\t\t\tif ( !float.IsFinite( sum ) )\n\t\t\t\t\t{\n\t\t\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t\t\t$\u0022[LLM:ERROR] {outputName} produced non-finite context at \u0022 \u002B\n\t\t\t\t\t\t\t$\u0022[{head},{query},{component}]: {sum:G9}.\u0022 );\n\t\t\t\t\t}\n\t\t\t\t\tint outputIndex =\n\t\t\t\t\t\t(head * queryLength \u002B query) * headDimension \u002B component;\n\t\t\t\t\toutput[outputIndex] = sum;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn new Tensor(\n\t\t\toutputName,\n\t\t\tnew[] { headCount, queryLength, headDimension },\n\t\t\toutput );\n\t}\n\n\tpublic static string DescribeAttentionContextElement(\n\t\tTensor probabilities,\n\t\tTensor valueHeads,\n\t\tTensor context,\n\t\tint head,\n\t\tint query,\n\t\tint component,\n\t\tfloat expected )\n\t{\n\t\tif ( probabilities is null || valueHeads is null || context is null ||\n\t\t\tprobabilities.Rank != 3 || valueHeads.Rank != 3 || context.Rank != 3 ||\n\t\t\tprobabilities.Shape[0] != valueHeads.Shape[0] ||\n\t\t\tprobabilities.Shape[0] != context.Shape[0] ||\n\t\t\tprobabilities.Shape[1] != context.Shape[1] ||\n\t\t\tprobabilities.Shape[2] != valueHeads.Shape[1] ||\n\t\t\tvalueHeads.Shape[2] != context.Shape[2] )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\u0022[LLM:ERROR] Attention-context diagnostic requires compatible \u0022 \u002B\n\t\t\t\t\u0022probability [head,query,key], V [head,key,component], and context \u0022 \u002B\n\t\t\t\t\u0022[head,query,component] tensors.\u0022 );\n\t\t}\n\t\tif ( head \u003C 0 || head \u003E= context.Shape[0] ||\n\t\t\tquery \u003C 0 || query \u003E= context.Shape[1] ||\n\t\t\tcomponent \u003C 0 || component \u003E= context.Shape[2] )\n\t\t{\n\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t$\u0022[LLM:ERROR] Attention-context diagnostic index \u0022 \u002B\n\t\t\t\t$\u0022[{head},{query},{component}] is outside {context.ShapeText}.\u0022 );\n\t\t}\n\n\t\tint keyLength = probabilities.Shape[2];\n\t\tint headDimension = valueHeads.Shape[2];\n\t\tint probabilityRow = (head * probabilities.Shape[1] \u002B query) * keyLength;\n\t\tfloat[] probabilityValues = new float[keyLength];\n\t\tfloat[] values = new float[keyLength];\n\t\tfloat[] products = new float[keyLength];\n\t\tfloat sum = 0;\n\t\tfor ( int key = 0; key \u003C keyLength; key\u002B\u002B )\n\t\t{\n\t\t\tprobabilityValues[key] = probabilities.Data[probabilityRow \u002B key];\n\t\t\tint valueIndex = (head * keyLength \u002B key) * headDimension \u002B component;\n\t\t\tvalues[key] = valueHeads.Data[valueIndex];\n\t\t\tproducts[key] = probabilityValues[key] * values[key];\n\t\t\tsum \u002B= products[key];\n\t\t}\n\n\t\tint contextIndex =\n\t\t\t(head * context.Shape[1] \u002B query) * context.Shape[2] \u002B component;\n\t\tfloat actual = context.Data[contextIndex];\n\t\tif ( BitConverter.SingleToInt32Bits( sum ) !=\n\t\t\tBitConverter.SingleToInt32Bits( actual ) )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] Attention-context diagnostic recomputation differs at \u0022 \u002B\n\t\t\t\t$\u0022[{head},{query},{component}]: recomputed={sum:G9}, actual={actual:G9}.\u0022 );\n\t\t}\n\n\t\treturn $\u0022head={head} query={query} component={component} \u0022 \u002B\n\t\t\t$\u0022probabilities={FormatFloatList( probabilityValues )} \u0022 \u002B\n\t\t\t$\u0022V={FormatFloatList( values )} products={FormatFloatList( products )} \u0022 \u002B\n\t\t\t$\u0022accumulated={actual:G9} python={expected:G9} \u0022 \u002B\n\t\t\t$\u0022abs_diff={MathF.Abs( actual - expected ):G12}\u0022;\n\t}\n\n\tpublic static string DescribeAttentionSoftmaxRow(\n\t\tTensor maskedScores,\n\t\tTensor probabilities,\n\t\tTensor expectedProbabilities,\n\t\tint head,\n\t\tint query )\n\t{\n\t\tif ( maskedScores is null || probabilities is null || expectedProbabilities is null ||\n\t\t\tmaskedScores.Rank != 3 || probabilities.Rank != 3 || expectedProbabilities.Rank != 3 ||\n\t\t\tmaskedScores.Shape[0] != probabilities.Shape[0] ||\n\t\t\tmaskedScores.Shape[1] != probabilities.Shape[1] ||\n\t\t\tmaskedScores.Shape[2] != probabilities.Shape[2] ||\n\t\t\tmaskedScores.Shape[0] != expectedProbabilities.Shape[0] ||\n\t\t\tmaskedScores.Shape[1] != expectedProbabilities.Shape[1] ||\n\t\t\tmaskedScores.Shape[2] != expectedProbabilities.Shape[2] )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\u0022[LLM:ERROR] Softmax row diagnostic requires matching rank-3 tensors.\u0022 );\n\t\t}\n\t\tif ( head \u003C 0 || head \u003E= maskedScores.Shape[0] ||\n\t\t\tquery \u003C 0 || query \u003E= maskedScores.Shape[1] )\n\t\t{\n\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t$\u0022[LLM:ERROR] Softmax row diagnostic index [{head},{query}] is outside \u0022 \u002B\n\t\t\t\t$\u0022{maskedScores.ShapeText}.\u0022 );\n\t\t}\n\n\t\tint keyLength = maskedScores.Shape[2];\n\t\tint row = (head * maskedScores.Shape[1] \u002B query) * keyLength;\n\t\tfloat rowMaximum = float.NegativeInfinity;\n\t\tfor ( int key = 0; key \u003C keyLength; key\u002B\u002B )\n\t\t{\n\t\t\trowMaximum = MathF.Max( rowMaximum, maskedScores.Data[row \u002B key] );\n\t\t}\n\n\t\tfloat[] inputs = new float[keyLength];\n\t\tfloat[] shifted = new float[keyLength];\n\t\tfloat[] exponentials = new float[keyLength];\n\t\tfloat[] actual = new float[keyLength];\n\t\tfloat[] expected = new float[keyLength];\n\t\tfloat exponentialSum = 0;\n\t\tfloat probabilitySum = 0;\n\t\tfor ( int key = 0; key \u003C keyLength; key\u002B\u002B )\n\t\t{\n\t\t\tinputs[key] = maskedScores.Data[row \u002B key];\n\t\t\tshifted[key] = inputs[key] - rowMaximum;\n\t\t\texponentials[key] = MathF.Exp( shifted[key] );\n\t\t\texponentialSum \u002B= exponentials[key];\n\t\t\tactual[key] = probabilities.Data[row \u002B key];\n\t\t\texpected[key] = expectedProbabilities.Data[row \u002B key];\n\t\t\tprobabilitySum \u002B= actual[key];\n\t\t}\n\n\t\treturn $\u0022head={head} query={query} input={FormatFloatList( inputs )} \u0022 \u002B\n\t\t\t$\u0022max={rowMaximum:G9} shifted={FormatFloatList( shifted )} \u0022 \u002B\n\t\t\t$\u0022exp={FormatFloatList( exponentials )} exp_sum={exponentialSum:G9} \u0022 \u002B\n\t\t\t$\u0022actual={FormatFloatList( actual )} python={FormatFloatList( expected )} \u0022 \u002B\n\t\t\t$\u0022row_sum={probabilitySum:G9}\u0022;\n\t}\n\n\tpublic static string DescribeAttentionMaskApplication(\n\t\tTensor unmaskedScores,\n\t\tTensor maskedScores,\n\t\tstring attentionType,\n\t\tfloat maskedSentinel,\n\t\tint head,\n\t\tint query,\n\t\tint key )\n\t{\n\t\tif ( unmaskedScores is null || maskedScores is null ||\n\t\t\tunmaskedScores.Rank != 3 || maskedScores.Rank != 3 ||\n\t\t\tunmaskedScores.Shape[0] != maskedScores.Shape[0] ||\n\t\t\tunmaskedScores.Shape[1] != maskedScores.Shape[1] ||\n\t\t\tunmaskedScores.Shape[2] != maskedScores.Shape[2] )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\u0022[LLM:ERROR] Mask diagnostic requires matching rank-3 unmasked/masked tensors.\u0022 );\n\t\t}\n\n\t\tint queryLength = unmaskedScores.Shape[1];\n\t\tint keyLength = unmaskedScores.Shape[2];\n\t\tif ( head \u003C 0 || head \u003E= unmaskedScores.Shape[0] )\n\t\t{\n\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t$\u0022[LLM:ERROR] Mask diagnostic head {head} is outside \u0022 \u002B\n\t\t\t\t$\u0022{unmaskedScores.ShapeText}.\u0022 );\n\t\t}\n\t\tbool allowed = IsLayer0AttentionAllowed(\n\t\t\tattentionType,\n\t\t\tquery,\n\t\t\tkey,\n\t\t\tqueryLength,\n\t\t\tkeyLength );\n\t\tint index = (head * queryLength \u002B query) * keyLength \u002B key;\n\t\tfloat unmasked = unmaskedScores.Data[index];\n\t\tfloat masked = maskedScores.Data[index];\n\t\treturn $\u0022head={head} query={query} key={key} allowed={allowed} \u0022 \u002B\n\t\t\t$\u0022unmasked={unmasked:G9} masked={masked:G9} \u0022 \u002B\n\t\t\t$\u0022masked_bits=0x{BitConverter.SingleToInt32Bits( masked ):X8} \u0022 \u002B\n\t\t\t$\u0022expected_sentinel={maskedSentinel:G9} \u0022 \u002B\n\t\t\t$\u0022sentinel_bits=0x{BitConverter.SingleToInt32Bits( maskedSentinel ):X8}\u0022;\n\t}\n\n\tpublic static string DescribeAttentionScore(\n\t\tTensor queryHeads,\n\t\tTensor keyHeads,\n\t\tTensor scores,\n\t\tint head,\n\t\tint query,\n\t\tint key,\n\t\tfloat scale,\n\t\tfloat expected )\n\t{\n\t\tif ( queryHeads is null || keyHeads is null || scores is null ||\n\t\t\tqueryHeads.Rank != 3 || keyHeads.Rank != 3 || scores.Rank != 3 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\u0022[LLM:ERROR] Attention-score diagnostic requires rank-3 Q, K, and score tensors.\u0022 );\n\t\t}\n\t\tint headDimension = queryHeads.Shape[2];\n\t\tif ( headDimension != 4 || keyHeads.Shape[2] != headDimension ||\n\t\t\tqueryHeads.Shape[0] != keyHeads.Shape[0] ||\n\t\t\tscores.Shape[0] != queryHeads.Shape[0] ||\n\t\t\tscores.Shape[1] != queryHeads.Shape[1] ||\n\t\t\tscores.Shape[2] != keyHeads.Shape[1] )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] Attention-score diagnostic expected compatible Q/K/scores with \u0022 \u002B\n\t\t\t\t$\u0022head dimension 4, found Q={queryHeads.ShapeText}, K={keyHeads.ShapeText}, \u0022 \u002B\n\t\t\t\t$\u0022scores={scores.ShapeText}.\u0022 );\n\t\t}\n\t\tif ( head \u003C 0 || head \u003E= scores.Shape[0] || query \u003C 0 || query \u003E= scores.Shape[1] ||\n\t\t\tkey \u003C 0 || key \u003E= scores.Shape[2] )\n\t\t{\n\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t$\u0022[LLM:ERROR] Attention-score diagnostic index [{head},{query},{key}] \u0022 \u002B\n\t\t\t\t$\u0022is outside {scores.ShapeText}.\u0022 );\n\t\t}\n\n\t\tint queryRow = (head * queryHeads.Shape[1] \u002B query) * headDimension;\n\t\tint keyRow = (head * keyHeads.Shape[1] \u002B key) * headDimension;\n\t\tfloat q0 = queryHeads.Data[queryRow];\n\t\tfloat q1 = queryHeads.Data[queryRow \u002B 1];\n\t\tfloat q2 = queryHeads.Data[queryRow \u002B 2];\n\t\tfloat q3 = queryHeads.Data[queryRow \u002B 3];\n\t\tfloat k0 = keyHeads.Data[keyRow];\n\t\tfloat k1 = keyHeads.Data[keyRow \u002B 1];\n\t\tfloat k2 = keyHeads.Data[keyRow \u002B 2];\n\t\tfloat k3 = keyHeads.Data[keyRow \u002B 3];\n\t\tfloat p0 = q0 * k0;\n\t\tfloat p1 = q1 * k1;\n\t\tfloat p2 = q2 * k2;\n\t\tfloat p3 = q3 * k3;\n\t\tfloat rawSum = ((p0 \u002B p1) \u002B p2) \u002B p3;\n\t\tfloat recomputed = scale == 1.0f ? rawSum : rawSum * scale;\n\t\tint scoreIndex = (head * scores.Shape[1] \u002B query) * scores.Shape[2] \u002B key;\n\t\tfloat actual = scores.Data[scoreIndex];\n\t\tif ( BitConverter.SingleToInt32Bits( recomputed ) !=\n\t\t\tBitConverter.SingleToInt32Bits( actual ) )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] Attention-score diagnostic recomputation differs at \u0022 \u002B\n\t\t\t\t$\u0022[{head},{query},{key}]: recomputed={recomputed:G9}, actual={actual:G9}.\u0022 );\n\t\t}\n\n\t\treturn $\u0022head={head} query={query} key={key} \u0022 \u002B\n\t\t\t$\u0022q=[{q0:G9},{q1:G9},{q2:G9},{q3:G9}] \u0022 \u002B\n\t\t\t$\u0022k=[{k0:G9},{k1:G9},{k2:G9},{k3:G9}] \u0022 \u002B\n\t\t\t$\u0022products=[{p0:G9},{p1:G9},{p2:G9},{p3:G9}] \u0022 \u002B\n\t\t\t$\u0022raw_sum={rawSum:G9} scale={scale:G9} final={actual:G9} expected={expected:G9}\u0022;\n\t}\n\n\tprivate static string FormatFloatList( IReadOnlyList\u003Cfloat\u003E values )\n\t{\n\t\tstring[] formatted = new string[values.Count];\n\t\tfor ( int index = 0; index \u003C values.Count; index\u002B\u002B )\n\t\t{\n\t\t\tformatted[index] = values[index].ToString( \u0022G9\u0022 );\n\t\t}\n\t\treturn $\u0022[{string.Join( \u0022,\u0022, formatted )}]\u0022;\n\t}\n}\n"},{"Ident":"jeffskitchen.llm_poc","Path":"Llm/TinyStoriesGreedyGenerator.cs","FileName":"TinyStoriesGreedyGenerator.cs","PackageType":"game","CodeKind":"Game","AssetVersionId":342521,"Code":"using Sandbox.Diagnostics;\n\nnamespace LlmPoc.Llm;\n\npublic sealed class GreedyGenerationStepResult\n{\n\tpublic int Step { get; init; }\n\tpublic int InputSequenceLength { get; init; }\n\tpublic int NewTokenPosition { get; init; }\n\tpublic int TokenId { get; init; }\n\tpublic string DecodedToken { get; init; }\n\tpublic float Top1Logit { get; init; }\n\tpublic int Top2TokenId { get; init; }\n\tpublic float Top2Logit { get; init; }\n\tpublic float Top1Top2Margin { get; init; }\n\tpublic LogitRank[] TopFive { get; init; }\n\tpublic bool EosReached { get; init; }\n\tpublic double ForwardMilliseconds { get; init; }\n\tpublic double LmHeadMilliseconds { get; init; }\n\tpublic double StepMilliseconds { get; init; }\n\tpublic double[] LayerMilliseconds { get; init; }\n}\n\npublic sealed class GreedyGenerationStepObservation\n{\n\tpublic GreedyGenerationStepResult Step { get; init; }\n\tpublic int[] InputTokenIds { get; init; }\n\tpublic TinyStoriesModelForwardResult Forward { get; init; }\n}\n\npublic sealed class GreedyGenerationResult\n{\n\tpublic int[] PromptTokenIds { get; init; }\n\tpublic int[] GeneratedTokenIds { get; init; }\n\tpublic int[] FullSequenceTokenIds { get; init; }\n\tpublic string GeneratedText { get; init; }\n\tpublic string StopReason { get; init; }\n\tpublic bool EosReached { get; init; }\n\tpublic GreedyGenerationStepResult[] Steps { get; init; }\n\tpublic double TotalForwardMilliseconds { get; init; }\n\tpublic double TotalGenerationMilliseconds { get; init; }\n}\n\n/// \u003Csummary\u003E\n/// Deterministic correctness baseline: every new token recomputes the complete\n/// model over the complete current context. No K/V state is retained.\n/// \u003C/summary\u003E\npublic static class TinyStoriesGreedyGenerator\n{\n\tpublic static GreedyGenerationResult Generate(\n\t\tSboxLlmModel model,\n\t\tTinyStoriesConfig config,\n\t\tGpt2ByteBpeTokenizer tokenizer,\n\t\tIReadOnlyList\u003Cint\u003E promptTokenIds,\n\t\tint maxNewTokens,\n\t\tAction\u003CGreedyGenerationStepObservation\u003E observer = null,\n\t\tbool logLifecycle = true )\n\t{\n\t\tif ( model is null ) throw new ArgumentNullException( nameof( model ) );\n\t\tif ( config is null ) throw new ArgumentNullException( nameof( config ) );\n\t\tif ( tokenizer is null ) throw new ArgumentNullException( nameof( tokenizer ) );\n\t\tif ( promptTokenIds is null ) throw new ArgumentNullException( nameof( promptTokenIds ) );\n\t\tif ( promptTokenIds.Count == 0 )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t\u0022[LLM:ERROR] Greedy generation requires at least one prompt token.\u0022,\n\t\t\t\tnameof( promptTokenIds ) );\n\t\t}\n\t\tif ( maxNewTokens \u003C= 0 )\n\t\t{\n\t\t\tthrow new ArgumentOutOfRangeException(\n\t\t\t\tnameof( maxNewTokens ), maxNewTokens,\n\t\t\t\t\u0022[LLM:ERROR] Greedy generation maxNewTokens must be positive.\u0022 );\n\t\t}\n\t\tif ( promptTokenIds.Count \u003E config.MaximumPositions - maxNewTokens )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] Greedy generation prompt length {promptTokenIds.Count} plus \u0022 \u002B\n\t\t\t\t$\u0022maxNewTokens {maxNewTokens} would exceed maximum context length \u0022 \u002B\n\t\t\t\t$\u0022{config.MaximumPositions}; context truncation is disabled.\u0022 );\n\t\t}\n\n\t\tint[] promptCopy = promptTokenIds.ToArray();\n\t\tList\u003Cint\u003E context = new( promptCopy.Length \u002B maxNewTokens );\n\t\tcontext.AddRange( promptCopy );\n\t\tList\u003Cint\u003E generated = new( maxNewTokens );\n\t\tList\u003CGreedyGenerationStepResult\u003E steps = new( maxNewTokens );\n\t\tdouble totalForwardMilliseconds = 0;\n\t\tdouble totalGenerationMilliseconds = 0;\n\t\tbool eosReached = false;\n\n\t\tif ( logLifecycle )\n\t\t{\n\t\t\tLlmLog.Info(\n\t\t\t\t\u0022GEN\u0022,\n\t\t\t\t$\u0022starting prompt_tokens={promptCopy.Length} max_new_tokens={maxNewTokens} \u0022 \u002B\n\t\t\t\t$\u0022maximum_context={config.MaximumPositions} strategy=greedy \u0022 \u002B\n\t\t\t\t\u0022full_recompute=true kv_cache=false\u0022 );\n\t\t}\n\n\t\tfor ( int stepIndex = 0; stepIndex \u003C maxNewTokens; stepIndex\u002B\u002B )\n\t\t{\n\t\t\tFastTimer stepTimer = FastTimer.StartNew();\n\t\t\tTinyStoriesModelForwardResult forward =\n\t\t\t\tTinyStoriesModelForward.ForwardLastTokenLogits( model, config, context );\n\t\t\tint tokenId = TinyStoriesModelHead.ArgmaxFinite(\n\t\t\t\tforward.Logits.Data, $\u0022generation.step{stepIndex}.logits\u0022 );\n\t\t\tLogitRank[] topFive = TinyStoriesModelHead.TopKFinite(\n\t\t\t\tforward.Logits.Data, 5, $\u0022generation.step{stepIndex}.logits\u0022 );\n\t\t\tstring decodedToken = tokenizer.Decode( new[] { tokenId } );\n\t\t\tfloat margin = topFive[0].Logit - topFive[1].Logit;\n\t\t\tbool stepEos = tokenId == config.EosTokenId;\n\t\t\tGreedyGenerationStepResult step = new()\n\t\t\t{\n\t\t\t\tStep = stepIndex,\n\t\t\t\tInputSequenceLength = context.Count,\n\t\t\t\tNewTokenPosition = context.Count,\n\t\t\t\tTokenId = tokenId,\n\t\t\t\tDecodedToken = decodedToken,\n\t\t\t\tTop1Logit = topFive[0].Logit,\n\t\t\t\tTop2TokenId = topFive[1].TokenId,\n\t\t\t\tTop2Logit = topFive[1].Logit,\n\t\t\t\tTop1Top2Margin = margin,\n\t\t\t\tTopFive = topFive,\n\t\t\t\tEosReached = stepEos,\n\t\t\t\tForwardMilliseconds = forward.TotalMilliseconds,\n\t\t\t\tLmHeadMilliseconds = forward.LmHeadMilliseconds,\n\t\t\t\tStepMilliseconds = stepTimer.ElapsedMilliSeconds,\n\t\t\t\tLayerMilliseconds = forward.LayerMilliseconds\n\t\t\t};\n\n\t\t\tGreedyGenerationStepObservation observation = new()\n\t\t\t{\n\t\t\t\tStep = step,\n\t\t\t\tInputTokenIds = context.ToArray(),\n\t\t\t\tForward = forward\n\t\t\t};\n\t\t\tobserver?.Invoke( observation );\n\t\t\tif ( observer is null \u0026\u0026 logLifecycle )\n\t\t\t{\n\t\t\t\tLlmLog.Info(\n\t\t\t\t\t\u0022GEN\u0022,\n\t\t\t\t\t$\u0022step={stepIndex} context={context.Count} token={tokenId} \u0022 \u002B\n\t\t\t\t\t$\u0022piece=\u0027{EscapeVisible( decodedToken )}\u0027 margin={margin:G9} \u0022 \u002B\n\t\t\t\t\t$\u0022forward_ms={forward.TotalMilliseconds:N4}\u0022 );\n\t\t\t}\n\t\t\tLlmLog.Trace(\n\t\t\t\t\u0022GEN\u0022,\n\t\t\t\t$\u0022step={stepIndex} sequence={context.Count} new_token_position={context.Count} \u0022 \u002B\n\t\t\t\t$\u0022top5=[{string.Join( \u0022,\u0022, topFive.Select( item =\u003E $\u0022{item.TokenId}:{item.Logit:G9}\u0022 ) )}]\u0022 );\n\n\t\t\tcontext.Add( tokenId );\n\t\t\tgenerated.Add( tokenId );\n\t\t\tsteps.Add( step );\n\t\t\ttotalForwardMilliseconds \u002B= forward.TotalMilliseconds;\n\t\t\ttotalGenerationMilliseconds \u002B= step.StepMilliseconds;\n\t\t\tif ( stepEos )\n\t\t\t{\n\t\t\t\teosReached = true;\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\n\t\tfor ( int index = 0; index \u003C promptCopy.Length; index\u002B\u002B )\n\t\t{\n\t\t\tif ( promptCopy[index] != promptTokenIds[index] )\n\t\t\t{\n\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t$\u0022[LLM:ERROR] Greedy generation mutated caller prompt token {index}.\u0022 );\n\t\t\t}\n\t\t}\n\n\t\treturn new GreedyGenerationResult\n\t\t{\n\t\t\tPromptTokenIds = promptCopy,\n\t\t\tGeneratedTokenIds = generated.ToArray(),\n\t\t\tFullSequenceTokenIds = context.ToArray(),\n\t\t\tGeneratedText = tokenizer.Decode( generated ),\n\t\t\tStopReason = eosReached ? \u0022eos\u0022 : \u0022max_new_tokens\u0022,\n\t\t\tEosReached = eosReached,\n\t\t\tSteps = steps.ToArray(),\n\t\t\tTotalForwardMilliseconds = totalForwardMilliseconds,\n\t\t\tTotalGenerationMilliseconds = totalGenerationMilliseconds\n\t\t};\n\t}\n\n\tprivate static string EscapeVisible( string value )\n\t{\n\t\treturn value\n\t\t\t.Replace( \u0022\\\\\u0022, \u0022\\\\\\\\\u0022 )\n\t\t\t.Replace( \u0022\\r\u0022, \u0022\\\\r\u0022 )\n\t\t\t.Replace( \u0022\\n\u0022, \u0022\\\\n\u0022 )\n\t\t\t.Replace( \u0022\\t\u0022, \u0022\\\\t\u0022 )\n\t\t\t.Replace( \u0022\u0027\u0022, \u0022\\\\\u0027\u0022 );\n\t}\n}\n"},{"Ident":"jeffskitchen.llm_poc","Path":"ui/button.cs.scss","FileName":"button.cs.scss","PackageType":"game","CodeKind":"Game","AssetVersionId":342521,"Code":".button\r\n{\r\n\tposition: relative;\r\n\t\r\n\t\u003E .button-right-column\r\n\t{\r\n\t\tflex-direction: column;\r\n\t}\r\n}\r\n\r\n//  default menu position is below\r\n.button-hover-menu\r\n{\r\n\tposition: absolute;\r\n\ttop: 100%;\r\n\tflex-direction: column;\r\n\r\n\t\u0026.hidden\r\n\t{\r\n\t\topacity: 0;\r\n\t\tpointer-events: none;\r\n\t}\r\n}"},{"Ident":"jeffskitchen.llm_poc","Path":"ui/dropdown.cs.scss","FileName":"dropdown.cs.scss","PackageType":"game","CodeKind":"Game","AssetVersionId":342521,"Code":".dropdown\r\n{\r\n\tgap: 2px;\r\n\tflex-grow: 1;\r\n\tcursor: pointer;\r\n\tjustify-content: flex-end;\r\n\talign-items: center;\r\n\tpadding: 0px 12px;\r\n\r\n\t.button-right-column\r\n\t{\r\n\t\tflex-grow: 1;\r\n\t}\r\n}\r\n"},{"Ident":"jeffskitchen.llm_poc","Path":"ui/components/packagelist.razor.scss","FileName":"packagelist.razor.scss","PackageType":"game","CodeKind":"Game","AssetVersionId":342521,"Code":".package-list\r\n{\r\n    flex-shrink: 1;\r\n    flex-wrap: wrap;\r\n    flex-grow: 1;\r\n\r\n    h1\r\n    {\r\n        width: 100%;\r\n        margin-top: 50px;\r\n        font-size: 40px;\r\n    }\r\n\r\n    PackageCard\r\n    {\r\n        \u0026:hover\r\n        {\r\n            sound-in: \u0022ui.button.over\u0022;\r\n        }\r\n    }\r\n\r\n    VirtualGrid\r\n    {\r\n        width: 100%;\r\n        height: 100%;\r\n\r\n        .cell\r\n        {\r\n            \r\n        }\r\n    }\r\n}"},{"Ident":"jeffskitchen.llm_poc","Path":"Llm/ReferenceFloatData.cs","FileName":"ReferenceFloatData.cs","PackageType":"game","CodeKind":"Game","AssetVersionId":342521,"Code":"namespace LlmPoc.Llm;\n\npublic static class ReferenceFloatData\n{\n\tpublic static float[] LoadFromMounted( string path, int expectedCount )\n\t{\n\t\tif ( !FileSystem.Mounted.FileExists( path ) )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] Reference FP32 file \u0027{path}\u0027 is missing from FileSystem.Mounted.\u0022 );\n\t\t}\n\n\t\tbyte[] bytes = FileSystem.Mounted.ReadAllBytes( path ).ToArray();\n\t\tif ( bytes.Length % sizeof( float ) != 0 )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] Reference FP32 file \u0027{path}\u0027 has {bytes.Length:N0} bytes, \u0022 \u002B\n\t\t\t\t\u0022which is not divisible by four.\u0022 );\n\t\t}\n\n\t\tint count = bytes.Length / sizeof( float );\n\t\tif ( count != expectedCount )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] Reference FP32 file \u0027{path}\u0027 expected {expectedCount:N0} values \u0022 \u002B\n\t\t\t\t$\u0022({expectedCount * sizeof( float ):N0} bytes), found {count:N0} values \u0022 \u002B\n\t\t\t\t$\u0022({bytes.Length:N0} bytes).\u0022 );\n\t\t}\n\n\t\tfloat[] values = new float[count];\n\t\tfor ( int index = 0; index \u003C count; index\u002B\u002B )\n\t\t{\n\t\t\tint offset = index * sizeof( float );\n\t\t\tuint bits = (uint)(\n\t\t\t\tbytes[offset]\n\t\t\t\t| (bytes[offset \u002B 1] \u003C\u003C 8)\n\t\t\t\t| (bytes[offset \u002B 2] \u003C\u003C 16)\n\t\t\t\t| (bytes[offset \u002B 3] \u003C\u003C 24) );\n\t\t\tvalues[index] = BitConverter.Int32BitsToSingle( unchecked( (int)bits ) );\n\t\t}\n\t\treturn values;\n\t}\n\n\tpublic static int ArgmaxFinite( ReadOnlySpan\u003Cfloat\u003E values, string logicalName )\n\t{\n\t\tif ( values.Length == 0 )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t$\u0022[LLM:ERROR] {logicalName} cannot be argmaxed because it is empty.\u0022 );\n\t\t}\n\n\t\tint bestIndex = 0;\n\t\tfloat bestValue = values[0];\n\t\tif ( !float.IsFinite( bestValue ) )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] {logicalName} contains non-finite value {bestValue} at index 0.\u0022 );\n\t\t}\n\n\t\tfor ( int index = 1; index \u003C values.Length; index\u002B\u002B )\n\t\t{\n\t\t\tfloat value = values[index];\n\t\t\tif ( !float.IsFinite( value ) )\n\t\t\t{\n\t\t\t\tthrow new InvalidOperationException(\n\t\t\t\t\t$\u0022[LLM:ERROR] {logicalName} contains non-finite value {value} at index {index}.\u0022 );\n\t\t\t}\n\t\t\tif ( value \u003E bestValue )\n\t\t\t{\n\t\t\t\tbestValue = value;\n\t\t\t\tbestIndex = index;\n\t\t\t}\n\t\t}\n\t\treturn bestIndex;\n\t}\n}\n"},{"Ident":"jeffskitchen.llm_poc","Path":"Llm/TinyStoriesModelForward.cs","FileName":"TinyStoriesModelForward.cs","PackageType":"game","CodeKind":"Game","AssetVersionId":342521,"Code":"using Sandbox.Diagnostics;\n\nnamespace LlmPoc.Llm;\n\npublic sealed class TinyStoriesModelForwardResult\n{\n\tpublic int SequenceLength { get; init; }\n\tpublic Tensor FinalLayerNorm { get; init; }\n\tpublic Tensor Logits { get; init; }\n\tpublic double EmbeddingMilliseconds { get; init; }\n\tpublic double[] LayerMilliseconds { get; init; }\n\tpublic string[] AttentionTypes { get; init; }\n\tpublic double AllLayersMilliseconds { get; init; }\n\tpublic double FinalLayerNormMilliseconds { get; init; }\n\tpublic double LmHeadMilliseconds { get; init; }\n\tpublic double TotalMilliseconds { get; init; }\n}\n\n/// \u003Csummary\u003E\n/// Production-oriented full-context forward orchestration. It deliberately reuses\n/// the parity-proven embedding, transformer-layer, final-norm, and LM-head kernels.\n/// \u003C/summary\u003E\npublic static class TinyStoriesModelForward\n{\n\tpublic static TinyStoriesModelForwardResult ForwardLastTokenLogits(\n\t\tSboxLlmModel model,\n\t\tTinyStoriesConfig config,\n\t\tIReadOnlyList\u003Cint\u003E tokenIds,\n\t\tbool logDiagnostics = false )\n\t{\n\t\tif ( model is null ) throw new ArgumentNullException( nameof( model ) );\n\t\tif ( config is null ) throw new ArgumentNullException( nameof( config ) );\n\t\tif ( tokenIds is null ) throw new ArgumentNullException( nameof( tokenIds ) );\n\t\tif ( tokenIds.Count == 0 )\n\t\t{\n\t\t\tthrow new ArgumentException(\n\t\t\t\t\u0022[LLM:ERROR] Model forward requires at least one token.\u0022, nameof( tokenIds ) );\n\t\t}\n\t\tif ( tokenIds.Count \u003E config.MaximumPositions )\n\t\t{\n\t\t\tthrow new InvalidOperationException(\n\t\t\t\t$\u0022[LLM:ERROR] Model forward sequence length {tokenIds.Count} exceeds \u0022 \u002B\n\t\t\t\t$\u0022maximum context length {config.MaximumPositions}; context truncation is disabled.\u0022 );\n\t\t}\n\t\tfor ( int index = 0; index \u003C tokenIds.Count; index\u002B\u002B )\n\t\t{\n\t\t\tint tokenId = tokenIds[index];\n\t\t\tif ( tokenId \u003C 0 || tokenId \u003E= config.VocabularySize )\n\t\t\t{\n\t\t\t\tthrow new IndexOutOfRangeException(\n\t\t\t\t\t$\u0022[LLM:ERROR] Model forward token ID {tokenId} at index {index} is outside \u0022 \u002B\n\t\t\t\t\t$\u0022[0,{config.VocabularySize}).\u0022 );\n\t\t\t}\n\t\t}\n\n\t\tFastTimer totalTimer = FastTimer.StartNew();\n\t\tFastTimer embeddingTimer = FastTimer.StartNew();\n\t\tTensor hiddenStates = TinyStoriesForwardStages.CombineEmbeddings(\n\t\t\tmodel, config, tokenIds, logDiagnostics );\n\t\tdouble embeddingMilliseconds = embeddingTimer.ElapsedMilliSeconds;\n\n\t\tdouble[] layerMilliseconds = new double[config.LayerCount];\n\t\tstring[] attentionTypes = new string[config.LayerCount];\n\t\tdouble allLayersMilliseconds = 0;\n\t\tfor ( int layerIndex = 0; layerIndex \u003C config.LayerCount; layerIndex\u002B\u002B )\n\t\t{\n\t\t\tGptNeoLayerForwardResult layer = GptNeoTransformerLayer.Forward(\n\t\t\t\tmodel,\n\t\t\t\tconfig,\n\t\t\t\thiddenStates,\n\t\t\t\tlayerIndex,\n\t\t\t\tlogDiagnostics,\n\t\t\t\tvalidateInputMutation: false );\n\t\t\thiddenStates = layer.Output;\n\t\t\tlayerMilliseconds[layerIndex] = layer.ElapsedMilliseconds;\n\t\t\tattentionTypes[layerIndex] = layer.AttentionType;\n\t\t\tallLayersMilliseconds \u002B= layer.ElapsedMilliseconds;\n\t\t}\n\n\t\tFastTimer finalLayerNormTimer = FastTimer.StartNew();\n\t\tTensor finalLayerNorm = TinyStoriesForwardStages.ApplyFinalLayerNorm(\n\t\t\tmodel, config, hiddenStates, \u0022forward.final_ln\u0022, logDiagnostics );\n\t\tdouble finalLayerNormMilliseconds = finalLayerNormTimer.ElapsedMilliSeconds;\n\n\t\tFastTimer lmHeadTimer = FastTimer.StartNew();\n\t\tTensor logits = TinyStoriesModelHead.ProjectLastPositionNoBias(\n\t\t\tfinalLayerNorm,\n\t\t\tmodel.GetRequiredTensor( \u0022lm_head.weight\u0022 ),\n\t\t\t\u0022forward.final_logits_last_position\u0022 );\n\t\tdouble lmHeadMilliseconds = lmHeadTimer.ElapsedMilliSeconds;\n\t\tlogits.RequireShape( config.VocabularySize );\n\n\t\tif ( logDiagnostics )\n\t\t{\n\t\t\tLlmLog.Trace(\n\t\t\t\t\u0022PERF\u0022,\n\t\t\t\t$\u0022stage=model_forward sequence={tokenIds.Count} embeddings_ms={embeddingMilliseconds:N4} \u0022 \u002B\n\t\t\t\t$\u0022layers_ms={allLayersMilliseconds:N4} final_ln_ms={finalLayerNormMilliseconds:N4} \u0022 \u002B\n\t\t\t\t$\u0022lm_head_ms={lmHeadMilliseconds:N4} total_ms={totalTimer.ElapsedMilliSeconds:N4}\u0022 );\n\t\t}\n\n\t\treturn new TinyStoriesModelForwardResult\n\t\t{\n\t\t\tSequenceLength = tokenIds.Count,\n\t\t\tFinalLayerNorm = finalLayerNorm,\n\t\t\tLogits = logits,\n\t\t\tEmbeddingMilliseconds = embeddingMilliseconds,\n\t\t\tLayerMilliseconds = layerMilliseconds,\n\t\t\tAttentionTypes = attentionTypes,\n\t\t\tAllLayersMilliseconds = allLayersMilliseconds,\n\t\t\tFinalLayerNormMilliseconds = finalLayerNormMilliseconds,\n\t\t\tLmHeadMilliseconds = lmHeadMilliseconds,\n\t\t\tTotalMilliseconds = totalTimer.ElapsedMilliSeconds\n\t\t};\n\t}\n}\n"},{"Ident":"jeffskitchen.llm_poc","Path":"ui/controls/switchcontrol.razor.scss","FileName":"switchcontrol.razor.scss","PackageType":"game","CodeKind":"Game","AssetVersionId":342521,"Code":"\r\n.switchcontrol\r\n{\r\n    flex-direction: row;\r\n    width: 100px;\r\n    min-height: 24px;\r\n    align-items: center;\r\n    cursor: pointer;\r\n\r\n    .switch-frame\r\n    {\r\n        flex-grow: 0;\r\n        flex-shrink: 1;\r\n        width: 48px;\r\n        height: 16px;\r\n        background-color: #fff1;\r\n        margin: 0px 5px;\r\n        align-items: center;\r\n        border-radius: 100px;\r\n        transition: all 0.4s linear;\r\n\r\n        .switch-inner\r\n        {\r\n            position: relative;\r\n            flex-grow: 0;\r\n            flex-shrink: 1;\r\n            background-color: #999;\r\n            width: 25px;\r\n            height: 25px;\r\n            border-radius: 100px;\r\n            left: 20%;\r\n            transform: translateX( -50% );\r\n            transition: all 0.3s ease-out;\r\n        }\r\n    }\r\n\r\n    \u0026.active\r\n    {\r\n        .switch-frame\r\n        {\r\n            background-color: #fffa;\r\n        }\r\n\r\n        .switch-inner\r\n        {\r\n            left: 80%;\r\n            background-color: #fff;\r\n        }\r\n    }\r\n}\r\n"},{"Ident":"jeffskitchen.llm_poc","Path":"styles/form/_dropdown.scss","FileName":"_dropdown.scss","PackageType":"game","CodeKind":"Game","AssetVersionId":342521,"Code":"\r\n$primary: red !default;\r\n$primary-alt: white !default;\r\n\r\n$switch-padding: 6px !default;\r\n\r\n.button.popupbutton.dropdown\r\n{\r\n\tcursor: pointer;\r\n\ttransition: all .1s ease-out;\r\n\tposition: relative;\r\n\r\n\t\u003E .dropdown_indicator\r\n\t{\r\n\t\tposition: absolute;\r\n\t\tright: 8px;\r\n\t}\r\n\r\n\t\u0026.open\r\n\t{\r\n\t\tborder-bottom-left-radius: 1px;\r\n\t\tborder-bottom-right-radius: 1px;\r\n\t\ttransition: border-radius 0.2s ease-out;\r\n\t}\r\n}\r\n\r\nselect\r\n{\r\n\tmin-height: 40px;\r\n\r\n\t\u003E option\r\n\t{\r\n\t\tdisplay: none;\r\n\t}\r\n}"},{"Ident":"jeffskitchen.llm_poc","Path":"ui/menupanel.razor.scss","FileName":"menupanel.razor.scss","PackageType":"game","CodeKind":"Game","AssetVersionId":342521,"Code":"\r\nmenupanel\r\n{\r\n    position: absolute;\r\n    z-index: 1000;\r\n    pointer-events: all;\r\n    font-size: 12px;\r\n    flex-shrink: 0;\r\n\r\n    .background\r\n    {\r\n        position: absolute;\r\n        left: -5000px;\r\n        right: -5000px;\r\n        top: -5000px;\r\n        bottom: -5000px;\r\n    }\r\n\r\n    \u003E .inner\r\n    {\r\n        min-width: 200px;\r\n        min-height: 20px;\r\n        flex-direction: column;\r\n        font-family: Poppins;\r\n        font-weight: bold;\r\n        border-radius: 10px;\r\n        box-shadow: 5px 5px 30px #000e;\r\n        background-color: #2a2a2a;\r\n        flex-shrink: 0;\r\n\r\n        .spacer\r\n        {\r\n            height: 1px;\r\n            background-color: #0005;\r\n        }\r\n\r\n        .option\r\n        {\r\n            color: #fffa;\r\n            padding: 0px 8px;\r\n            cursor: pointer;\r\n            flex-shrink: 0;\r\n            height: 32px;\r\n\r\n            \u0026:first-child\r\n            {\r\n                border-top-left-radius: 10px;\r\n                border-top-right-radius: 10px;\r\n            }\r\n\r\n            \u0026:last-child\r\n            {\r\n                border-bottom-left-radius: 10px;\r\n                border-bottom-right-radius: 10px;\r\n            }\r\n\r\n            .icon\r\n            {\r\n                padding: 8px;\r\n                font-family: Material Icons;\r\n                justify-content: center;\r\n                align-items: center;\r\n                flex-shrink: 0;\r\n            }\r\n\r\n            .text\r\n            {\r\n                padding: 8px;\r\n                flex-shrink: 0;\r\n            }\r\n\r\n            \u0026:hover\r\n            {\r\n                background-color: #3472e6;\r\n                color: #f5f8fe;\r\n            }\r\n        }\r\n    }\r\n}\r\n"}]}