clark-labs commited on
Commit
3e86fb6
·
verified ·
1 Parent(s): 2fa33a2

Publish RGB realtime UI video x2 model and corrected evidence

Browse files
.gitattributes CHANGED
@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ examples/product-demo-1-before-after.png filter=lfs diff=lfs merge=lfs -text
37
+ examples/product-demo-2-before-after.png filter=lfs diff=lfs merge=lfs -text
38
+ examples/product-demo-3-before-after.png filter=lfs diff=lfs merge=lfs -text
39
+ examples/product-demo-before-after.png filter=lfs diff=lfs merge=lfs -text
LICENSE ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+
9
+ "License" shall mean the terms and conditions for use, reproduction,
10
+ and distribution as defined by Sections 1 through 9 of this document.
11
+
12
+ "Licensor" shall mean the copyright owner or entity authorized by
13
+ the copyright owner that is granting the License.
14
+
15
+ "Legal Entity" shall mean the union of the acting entity and all
16
+ other entities that control, are controlled by, or are under common
17
+ control with that entity. For the purposes of this definition,
18
+ "control" means (i) the power, direct or indirect, to cause the
19
+ direction or management of such entity, whether by contract or
20
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
+ outstanding shares, or (iii) beneficial ownership of such entity.
22
+
23
+ "You" (or "Your") shall mean an individual or Legal Entity
24
+ exercising permissions granted by this License.
25
+
26
+ "Source" form shall mean the preferred form for making modifications,
27
+ including but not limited to software source code, documentation
28
+ source, and configuration files.
29
+
30
+ "Object" form shall mean any form resulting from mechanical
31
+ transformation or translation of a Source form, including but
32
+ not limited to compiled object code, generated documentation,
33
+ and conversions to other media types.
34
+
35
+ "Work" shall mean the work of authorship, whether in Source or
36
+ Object form, made available under the License, as indicated by a
37
+ copyright notice that is included in or attached to the work
38
+ (an example is provided in the Appendix below).
39
+
40
+ "Derivative Works" shall mean any work, whether in Source or Object
41
+ form, that is based on (or derived from) the Work and for which the
42
+ editorial revisions, annotations, elaborations, or other modifications
43
+ represent, as a whole, an original work of authorship. For the purposes
44
+ of this License, Derivative Works shall not include works that remain
45
+ separable from, or merely link (or bind by name) to the interfaces of,
46
+ the Work and Derivative Works thereof.
47
+
48
+ "Contribution" shall mean any work of authorship, including
49
+ the original version of the Work and any modifications or additions
50
+ to that Work or Derivative Works thereof, that is intentionally
51
+ submitted to Licensor for inclusion in the Work by the copyright owner
52
+ or by an individual or Legal Entity authorized to submit on behalf of
53
+ the copyright owner. For the purposes of this definition, "submitted"
54
+ means any form of electronic, verbal, or written communication sent
55
+ to the Licensor or its representatives, including but not limited to
56
+ communication on electronic mailing lists, source code control systems,
57
+ and issue tracking systems that are managed by, or on behalf of, the
58
+ Licensor for the purpose of discussing and improving the Work, but
59
+ excluding communication that is conspicuously marked or otherwise
60
+ designated in writing by the copyright owner as "Not a Contribution."
61
+
62
+ "Contributor" shall mean Licensor and any individual or Legal Entity
63
+ on behalf of whom a Contribution has been received by Licensor and
64
+ subsequently incorporated within the Work.
65
+
66
+ 2. Grant of Copyright License. Subject to the terms and conditions of
67
+ this License, each Contributor hereby grants to You a perpetual,
68
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69
+ copyright license to reproduce, prepare Derivative Works of,
70
+ publicly display, publicly perform, sublicense, and distribute the
71
+ Work and such Derivative Works in Source or Object form.
72
+
73
+ 3. Grant of Patent License. Subject to the terms and conditions of
74
+ this License, each Contributor hereby grants to You a perpetual,
75
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76
+ (except as stated in this section) patent license to make, have made,
77
+ use, offer to sell, sell, import, and otherwise transfer the Work,
78
+ where such license applies only to those patent claims licensable
79
+ by such Contributor that are necessarily infringed by their
80
+ Contribution(s) alone or by combination of their Contribution(s)
81
+ with the Work to which such Contribution(s) was submitted. If You
82
+ institute patent litigation against any entity (including a
83
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84
+ or a Contribution incorporated within the Work constitutes direct
85
+ or contributory patent infringement, then any patent licenses
86
+ granted to You under this License for that Work shall terminate
87
+ as of the date such litigation is filed.
88
+
89
+ 4. Redistribution. You may reproduce and distribute copies of the
90
+ Work or Derivative Works thereof in any medium, with or without
91
+ modifications, and in Source or Object form, provided that You
92
+ meet the following conditions:
93
+
94
+ (a) You must give any other recipients of the Work or
95
+ Derivative Works a copy of this License; and
96
+
97
+ (b) You must cause any modified files to carry prominent notices
98
+ stating that You changed the files; and
99
+
100
+ (c) You must retain, in the Source form of any Derivative Works
101
+ that You distribute, all copyright, patent, trademark, and
102
+ attribution notices from the Source form of the Work,
103
+ excluding those notices that do not pertain to any part of
104
+ the Derivative Works; and
105
+
106
+ (d) If the Work includes a "NOTICE" text file as part of its
107
+ distribution, then any Derivative Works that You distribute must
108
+ include a readable copy of the attribution notices contained
109
+ within such NOTICE file, excluding those notices that do not
110
+ pertain to any part of the Derivative Works, in at least one
111
+ of the following places: within a NOTICE text file distributed
112
+ as part of the Derivative Works; within the Source form or
113
+ documentation, if provided along with the Derivative Works; or,
114
+ within a display generated by the Derivative Works, if and
115
+ wherever such third-party notices normally appear. The contents
116
+ of the NOTICE file are for informational purposes only and
117
+ do not modify the License. You may add Your own attribution
118
+ notices within Derivative Works that You distribute, alongside
119
+ or as an addendum to the NOTICE text from the Work, provided
120
+ that such additional attribution notices cannot be construed
121
+ as modifying the License.
122
+
123
+ You may add Your own copyright statement to Your modifications and
124
+ may provide additional or different license terms and conditions
125
+ for use, reproduction, or distribution of Your modifications, or
126
+ for any such Derivative Works as a whole, provided Your use,
127
+ reproduction, and distribution of the Work otherwise complies with
128
+ the conditions stated in this License.
129
+
130
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131
+ any Contribution intentionally submitted for inclusion in the Work
132
+ by You to the Licensor shall be under the terms and conditions of
133
+ this License, without any additional terms or conditions.
134
+ Notwithstanding the above, nothing herein shall supersede or modify
135
+ the terms of any separate license agreement you may have executed
136
+ with Licensor regarding such Contributions.
137
+
138
+ 6. Trademarks. This License does not grant permission to use the trade
139
+ names, trademarks, service marks, or product names of the Licensor,
140
+ except as required for reasonable and customary use in describing the
141
+ origin of the Work and reproducing the content of the NOTICE file.
142
+
143
+ 7. Disclaimer of Warranty. Unless required by applicable law or
144
+ agreed to in writing, Licensor provides the Work (and each
145
+ Contributor provides its Contributions) on an "AS IS" BASIS,
146
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147
+ implied, including, without limitation, any warranties or conditions
148
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149
+ PARTICULAR PURPOSE. You are solely responsible for determining the
150
+ appropriateness of using or redistributing the Work and assume any
151
+ risks associated with Your exercise of permissions under this License.
152
+
153
+ 8. Limitation of Liability. In no event and under no legal theory,
154
+ whether in tort (including negligence), contract, or otherwise,
155
+ unless required by applicable law (such as deliberate and grossly
156
+ negligent acts) or agreed to in writing, shall any Contributor be
157
+ liable to You for damages, including any direct, indirect, special,
158
+ incidental, or consequential damages of any character arising as a
159
+ result of this License or out of the use or inability to use the
160
+ Work (including but not limited to damages for loss of goodwill,
161
+ work stoppage, computer failure or malfunction, or any and all
162
+ other commercial damages or losses), even if such Contributor
163
+ has been advised of the possibility of such damages.
164
+
165
+ 9. Accepting Warranty or Additional Liability. While redistributing
166
+ the Work or Derivative Works thereof, You may choose to offer,
167
+ and charge a fee for, acceptance of support, warranty, indemnity,
168
+ or other liability obligations and/or rights consistent with this
169
+ License. However, in accepting such obligations, You may act only
170
+ on Your own behalf and on Your sole responsibility, not on behalf
171
+ of any other Contributor, and only if You agree to indemnify,
172
+ defend, and hold each Contributor harmless for any liability
173
+ incurred by, or claims asserted against, such Contributor by reason
174
+ of your accepting any such warranty or additional liability.
175
+
176
+ END OF TERMS AND CONDITIONS
177
+
178
+ APPENDIX: How to apply the Apache License to your work.
179
+
180
+ To apply the Apache License to your work, attach the following
181
+ boilerplate notice, with the fields enclosed by brackets "[]"
182
+ replaced with your own identifying information. (Don't include
183
+ the brackets!) The text should be enclosed in the appropriate
184
+ comment syntax for the file format. We also recommend that a
185
+ file or class name and description of purpose be included on the
186
+ same "printed page" as the copyright notice for easier
187
+ identification within third-party archives.
188
+
189
+ Copyright [2021] [SwinIR Authors]
190
+
191
+ Licensed under the Apache License, Version 2.0 (the "License");
192
+ you may not use this file except in compliance with the License.
193
+ You may obtain a copy of the License at
194
+
195
+ http://www.apache.org/licenses/LICENSE-2.0
196
+
197
+ Unless required by applicable law or agreed to in writing, software
198
+ distributed under the License is distributed on an "AS IS" BASIS,
199
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
+ See the License for the specific language governing permissions and
201
+ limitations under the License.
README.md ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ library_name: custom
4
+ pipeline_tag: image-to-image
5
+ tags:
6
+ - super-resolution
7
+ - video
8
+ - webgpu
9
+ - browser
10
+ - product-demo
11
+ - ui
12
+ ---
13
+
14
+ # Clark Realtime UI Video Upscaler x2
15
+
16
+ A 16.6 KiB RGB x2 super-resolution model for product-demo screenshots and video. It is designed for UI edges, small text, buttons, panels, and code—not portraits or generative detail.
17
+
18
+ The browser artifact has 8,496 parameters and runs as three fused FP16 WebGPU residual blocks. BackdropKit uses it locally for eligible video previews and exports, with ordinary high-quality browser scaling as the automatic fallback when `shader-f16` WebGPU is unavailable.
19
+
20
+ ## Before and after
21
+
22
+ ![Actual low-resolution input, bicubic x2, and Clark realtime x2](examples/product-demo-before-after.png)
23
+
24
+ The left column is the actual low-resolution source before either scaler is applied. The middle and right columns both start from that exact source. A held-out high-resolution frame is used only for PSNR measurement; it is not shown or mislabeled as the input.
25
+
26
+ On three fixed frames from real Clark product-demo videos:
27
+
28
+ | Measure | Bicubic x2 | Clark x2 | Gain |
29
+ |---|---:|---:|---:|
30
+ | RGB PSNR | 32.307 dB | 33.570 dB | +1.263 dB |
31
+ | Edge PSNR | 30.476 dB | 31.666 dB | +1.191 dB |
32
+
33
+ Frame timestamps and deterministic crop coordinates are in `technical/example-evaluation.json`. Crops are selected by reference edge density only, never by model score.
34
+
35
+ ## Model
36
+
37
+ - Input: even-sized RGB image or video frame
38
+ - Output: RGB at 2x width and height
39
+ - Working grid: 2x RGB pixel-unshuffle
40
+ - Body: 12 channels, 3 residual blocks, 2 convolutions per block
41
+ - Output: 4x pixel-shuffle from the half-resolution working grid
42
+ - Browser weights: FP16, 16,992 bytes
43
+ - Browser SHA-256: `c6b4f22d8c38270225b809e97d233eb68d7a48eef17595ac17a60b4540c72dce`
44
+ - Safetensors: 34,712 bytes
45
+
46
+ Training used 206,232 product, UI, and multilingual text images, with 268 held-out images. The RGB refinement ran for 5,000 steps from an exactly equivalent luma initialization.
47
+
48
+ ## Measured browser performance
49
+
50
+ On an Apple M5 in Chrome 150 WebGPU, 1920x1080 to 3840x2160:
51
+
52
+ - Neural render: 8.9 ms median, 10.6 ms p95 over 120 measured frames
53
+ - Neural render plus H.264 enqueue: 9.4 ms median, 11.0 ms p95
54
+ - H.264: 120/120 output chunks, maximum queue 2, no encoder error
55
+
56
+ These measurements show compute headroom; they are not a promise of 60 displayed or exported frames per second on every browser. Decode cadence, browser scheduling, GPU, thermals, encoder, and export composition remain platform-dependent. The full receipt is in `technical/runtime-receipt.json`.
57
+
58
+ ## Browser use
59
+
60
+ Host `browser/model.bin`, `browser/realtimeVideoUpscaler.js`, and `browser/runtimeCapabilities.js` together. Then pass the hosted model URL explicitly:
61
+
62
+ ```js
63
+ import {
64
+ createRealtimeVideoUpscaler,
65
+ REALTIME_VIDEO_MODEL,
66
+ } from "./browser/realtimeVideoUpscaler.js";
67
+
68
+ const runtime = await createRealtimeVideoUpscaler({
69
+ width: video.videoWidth,
70
+ height: video.videoHeight,
71
+ model: {
72
+ ...REALTIME_VIDEO_MODEL,
73
+ url: "./browser/model.bin",
74
+ },
75
+ });
76
+
77
+ const enhancedCanvas = await runtime?.enhance(video);
78
+ // Draw enhancedCanvas into your composition or encoder.
79
+ runtime?.destroy();
80
+ ```
81
+
82
+ The fast neural path requires WebGPU plus `shader-f16`. Applications should preserve a standard canvas/video scaling path when those capabilities are absent.
83
+
84
+ ## Package contents
85
+
86
+ - `browser/model.bin`: exact FP16 shader payload
87
+ - `browser/model.json`: offsets, sizes, hash, and architecture ledger
88
+ - `browser/realtimeVideoUpscaler.js`: fused browser runtime
89
+ - `model.safetensors`: framework-friendly weights
90
+ - `realtime_student.py`: PyTorch architecture
91
+ - `examples/`: real product-demo comparisons
92
+ - `technical/`: evaluation, runtime, and package receipts
93
+
94
+ Built by Clark Labs for [BackdropKit](https://backdropkit.com/).
browser/model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c6b4f22d8c38270225b809e97d233eb68d7a48eef17595ac17a60b4540c72dce
3
+ size 16992
browser/model.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "backdropkit-realtime-luma-student-f16-v1",
3
+ "channels": 12,
4
+ "blocks": 3,
5
+ "residual_scale": 1.0,
6
+ "color_mode": "rgb",
7
+ "parameters": 8496,
8
+ "bytes": 16992,
9
+ "tensors": {
10
+ "input_projection": {
11
+ "offset": 0,
12
+ "bytes": 288,
13
+ "elements": 144
14
+ },
15
+ "block_0": {
16
+ "offset": 288,
17
+ "bytes": 5184,
18
+ "elements": 2592
19
+ },
20
+ "block_1": {
21
+ "offset": 5472,
22
+ "bytes": 5184,
23
+ "elements": 2592
24
+ },
25
+ "block_2": {
26
+ "offset": 10656,
27
+ "bytes": 5184,
28
+ "elements": 2592
29
+ },
30
+ "output_projection": {
31
+ "offset": 15840,
32
+ "bytes": 1152,
33
+ "elements": 576
34
+ }
35
+ },
36
+ "sha256": "c6b4f22d8c38270225b809e97d233eb68d7a48eef17595ac17a60b4540c72dce",
37
+ "source_checkpoint_sha256": "3cbb8a3358bbeaa239c517f1171dbb7706c2d453c961d17a4e6166aefde8c692"
38
+ }
browser/realtimeVideoUpscaler.js ADDED
@@ -0,0 +1,393 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { requestBestWebGpuAdapter } from "./runtimeCapabilities.js";
2
+
3
+ export const REALTIME_VIDEO_MODEL = Object.freeze({
4
+ blocks: 3,
5
+ bytes: 16992,
6
+ channels: 12,
7
+ colorMode: "rgb",
8
+ id: "clark-realtime-ui-video-x2",
9
+ scale: 2,
10
+ sha256: "c6b4f22d8c38270225b809e97d233eb68d7a48eef17595ac17a60b4540c72dce",
11
+ url: "/models/clark-realtime-ui-video-x2/model.bin?v=c6b4f22d",
12
+ });
13
+
14
+ export function realtimeVideoIsEligible({ enabled = true, height, kind, width }) {
15
+ return Boolean(
16
+ enabled
17
+ && kind === "video"
18
+ && Number(width) >= 64
19
+ && Number(height) >= 64
20
+ && width <= 1920
21
+ && height <= 1080
22
+ && width % 2 === 0
23
+ && height % 2 === 0
24
+ );
25
+ }
26
+
27
+ async function digestHex(bytes) {
28
+ if (!globalThis.crypto?.subtle) return null;
29
+ const digest = await crypto.subtle.digest("SHA-256", bytes);
30
+ return [...new Uint8Array(digest)]
31
+ .map((value) => value.toString(16).padStart(2, "0"))
32
+ .join("");
33
+ }
34
+
35
+ async function loadModel(model) {
36
+ const response = await fetch(model.url, { cache: "force-cache" });
37
+ if (!response.ok) throw new Error("Realtime video model download failed");
38
+ const payload = await response.arrayBuffer();
39
+ if (payload.byteLength !== model.bytes) {
40
+ throw new Error("Realtime video model size check failed");
41
+ }
42
+ const hash = await digestHex(payload);
43
+ if (hash && hash !== model.sha256) {
44
+ throw new Error("Realtime video model identity check failed");
45
+ }
46
+ return payload;
47
+ }
48
+
49
+ function mappedBuffer(device, values, usage) {
50
+ const bytes = values instanceof ArrayBuffer
51
+ ? new Uint8Array(values)
52
+ : new Uint8Array(values.buffer, values.byteOffset, values.byteLength);
53
+ const buffer = device.createBuffer({
54
+ mappedAtCreation: true,
55
+ size: Math.ceil(bytes.byteLength / 4) * 4,
56
+ usage,
57
+ });
58
+ new Uint8Array(buffer.getMappedRange()).set(bytes);
59
+ buffer.unmap();
60
+ return buffer;
61
+ }
62
+
63
+ function modelBuffer(device, payload, offset, bytes) {
64
+ return mappedBuffer(
65
+ device,
66
+ new Uint8Array(payload, offset, bytes),
67
+ GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
68
+ );
69
+ }
70
+
71
+ async function compiledModule(device, code) {
72
+ const module = device.createShaderModule({ code });
73
+ const compilation = await module.getCompilationInfo();
74
+ const errors = compilation.messages.filter((message) => message.type === "error");
75
+ if (errors.length) throw new Error(errors.map((message) => message.message).join("\n"));
76
+ return module;
77
+ }
78
+
79
+ function inputShader() {
80
+ return /* wgsl */ `
81
+ enable f16;
82
+ struct Dimensions { width: u32, height: u32, channels: u32, padding: u32 }
83
+ @group(0) @binding(0) var source: texture_external;
84
+ @group(0) @binding(1) var source_sampler: sampler;
85
+ @group(0) @binding(2) var<storage, read_write> output: array<f16>;
86
+ @group(0) @binding(3) var<storage, read> weights: array<f16>;
87
+ @group(0) @binding(4) var<uniform> dimensions: Dimensions;
88
+
89
+ @compute @workgroup_size(8, 8, 1)
90
+ fn main(@builtin(global_invocation_id) id: vec3<u32>) {
91
+ if (id.x >= dimensions.width || id.y >= dimensions.height) { return; }
92
+ let source_size = vec2<f32>(f32(dimensions.width * 2u), f32(dimensions.height * 2u));
93
+ let base = vec2<f32>(f32(id.x * 2u), f32(id.y * 2u));
94
+ let p0 = textureSampleBaseClampToEdge(source, source_sampler, (base + vec2<f32>(0.5, 0.5)) / source_size).rgb;
95
+ let p1 = textureSampleBaseClampToEdge(source, source_sampler, (base + vec2<f32>(1.5, 0.5)) / source_size).rgb;
96
+ let p2 = textureSampleBaseClampToEdge(source, source_sampler, (base + vec2<f32>(0.5, 1.5)) / source_size).rgb;
97
+ let p3 = textureSampleBaseClampToEdge(source, source_sampler, (base + vec2<f32>(1.5, 1.5)) / source_size).rgb;
98
+ let pixels = array<f32, 12>(
99
+ p0.r, p1.r, p2.r, p3.r,
100
+ p0.g, p1.g, p2.g, p3.g,
101
+ p0.b, p1.b, p2.b, p3.b,
102
+ );
103
+ for (var channel = 0u; channel < dimensions.channels; channel += 1u) {
104
+ let offset = channel * 12u;
105
+ var value = 0.0;
106
+ for (var input_channel = 0u; input_channel < 12u; input_channel += 1u) {
107
+ value += pixels[input_channel] * f32(weights[offset + input_channel]);
108
+ }
109
+ output[(id.y * dimensions.width + id.x) * dimensions.channels + channel] = f16(max(value, 0.0));
110
+ }
111
+ }
112
+ `;
113
+ }
114
+
115
+ function fusedShader(channels) {
116
+ const tile = 8;
117
+ const inputTile = tile + 4;
118
+ const intermediateTile = tile + 2;
119
+ const inputElements = inputTile * inputTile * channels;
120
+ const intermediateElements = intermediateTile * intermediateTile * channels;
121
+ const weightElements = 9 * channels * channels;
122
+ return /* wgsl */ `
123
+ enable f16;
124
+ struct Dimensions { width: u32, height: u32, channels: u32, padding: u32 }
125
+ @group(0) @binding(0) var<storage, read> input: array<f16>;
126
+ @group(0) @binding(1) var<storage, read> weights: array<f16>;
127
+ @group(0) @binding(2) var<storage, read_write> output: array<f16>;
128
+ @group(0) @binding(3) var<uniform> dimensions: Dimensions;
129
+ var<workgroup> input_tile: array<f16, ${inputElements}>;
130
+ var<workgroup> intermediate_tile: array<f16, ${intermediateElements}>;
131
+
132
+ fn input_index(x: u32, y: u32, channel: u32) -> u32 {
133
+ return (y * ${inputTile}u + x) * ${channels}u + channel;
134
+ }
135
+ fn intermediate_index(x: u32, y: u32, channel: u32) -> u32 {
136
+ return (y * ${intermediateTile}u + x) * ${channels}u + channel;
137
+ }
138
+ fn weight_index(output_channel: u32, kernel_x: u32, kernel_y: u32, input_channel: u32) -> u32 {
139
+ return ((output_channel * 3u + kernel_y) * 3u + kernel_x) * ${channels}u + input_channel;
140
+ }
141
+
142
+ @compute @workgroup_size(${tile}, ${tile}, 1)
143
+ fn main(
144
+ @builtin(workgroup_id) group: vec3<u32>,
145
+ @builtin(local_invocation_index) lane: u32,
146
+ @builtin(local_invocation_id) local: vec3<u32>,
147
+ ) {
148
+ let base_x = i32(group.x * ${tile}u);
149
+ let base_y = i32(group.y * ${tile}u);
150
+ for (var linear = lane; linear < ${inputElements}u; linear += ${tile * tile}u) {
151
+ let channel = linear % ${channels}u;
152
+ let pixel = linear / ${channels}u;
153
+ let tile_x = pixel % ${inputTile}u;
154
+ let tile_y = pixel / ${inputTile}u;
155
+ let source_x = base_x + i32(tile_x) - 2;
156
+ let source_y = base_y + i32(tile_y) - 2;
157
+ let inside = source_x >= 0 && source_y >= 0
158
+ && source_x < i32(dimensions.width) && source_y < i32(dimensions.height);
159
+ var value = 0.0h;
160
+ if (inside) {
161
+ value = input[(u32(source_y) * dimensions.width + u32(source_x)) * ${channels}u + channel];
162
+ }
163
+ input_tile[linear] = value;
164
+ }
165
+ workgroupBarrier();
166
+
167
+ for (var linear = lane; linear < ${intermediateElements}u; linear += ${tile * tile}u) {
168
+ let output_channel = linear % ${channels}u;
169
+ let pixel = linear / ${channels}u;
170
+ let tile_x = pixel % ${intermediateTile}u;
171
+ let tile_y = pixel / ${intermediateTile}u;
172
+ let output_x = base_x + i32(tile_x) - 1;
173
+ let output_y = base_y + i32(tile_y) - 1;
174
+ var sum = 0.0h;
175
+ if (output_x >= 0 && output_y >= 0
176
+ && output_x < i32(dimensions.width) && output_y < i32(dimensions.height)) {
177
+ for (var kernel_y = 0u; kernel_y < 3u; kernel_y += 1u) {
178
+ for (var kernel_x = 0u; kernel_x < 3u; kernel_x += 1u) {
179
+ for (var input_channel = 0u; input_channel < ${channels}u; input_channel += 1u) {
180
+ sum += input_tile[input_index(tile_x + kernel_x, tile_y + kernel_y, input_channel)]
181
+ * weights[weight_index(output_channel, kernel_x, kernel_y, input_channel)];
182
+ }
183
+ }
184
+ }
185
+ }
186
+ intermediate_tile[linear] = max(sum, 0.0h);
187
+ }
188
+ workgroupBarrier();
189
+
190
+ let output_x = group.x * ${tile}u + local.x;
191
+ let output_y = group.y * ${tile}u + local.y;
192
+ if (output_x >= dimensions.width || output_y >= dimensions.height) { return; }
193
+ for (var output_channel = 0u; output_channel < ${channels}u; output_channel += 1u) {
194
+ var sum = 0.0h;
195
+ for (var kernel_y = 0u; kernel_y < 3u; kernel_y += 1u) {
196
+ for (var kernel_x = 0u; kernel_x < 3u; kernel_x += 1u) {
197
+ for (var input_channel = 0u; input_channel < ${channels}u; input_channel += 1u) {
198
+ sum += intermediate_tile[intermediate_index(local.x + kernel_x, local.y + kernel_y, input_channel)]
199
+ * weights[${weightElements}u + weight_index(output_channel, kernel_x, kernel_y, input_channel)];
200
+ }
201
+ }
202
+ }
203
+ let residual = input_tile[input_index(local.x + 2u, local.y + 2u, output_channel)];
204
+ output[(output_y * dimensions.width + output_x) * ${channels}u + output_channel] = max(sum + residual, 0.0h);
205
+ }
206
+ }
207
+ `;
208
+ }
209
+
210
+ function outputShader() {
211
+ return /* wgsl */ `
212
+ enable f16;
213
+ struct Dimensions { width: u32, height: u32, channels: u32, padding: u32 }
214
+ @group(0) @binding(0) var source: texture_external;
215
+ @group(0) @binding(1) var source_sampler: sampler;
216
+ @group(0) @binding(2) var<storage, read> features: array<f16>;
217
+ @group(0) @binding(3) var<storage, read> weights: array<f16>;
218
+ @group(0) @binding(4) var<uniform> dimensions: Dimensions;
219
+
220
+ @vertex
221
+ fn vertex_main(@builtin(vertex_index) index: u32) -> @builtin(position) vec4<f32> {
222
+ let position = array<vec2<f32>, 3>(
223
+ vec2<f32>(-1.0, -1.0), vec2<f32>(3.0, -1.0), vec2<f32>(-1.0, 3.0)
224
+ );
225
+ return vec4<f32>(position[index], 0.0, 1.0);
226
+ }
227
+
228
+ @fragment
229
+ fn fragment_main(@builtin(position) position: vec4<f32>) -> @location(0) vec4<f32> {
230
+ let output_pixel = vec2<u32>(position.xy);
231
+ let work_pixel = output_pixel / 4u;
232
+ let subpixel = (output_pixel.y % 4u) * 4u + output_pixel.x % 4u;
233
+ let feature_offset = (work_pixel.y * dimensions.width + work_pixel.x) * dimensions.channels;
234
+ var residual = vec3<f32>(0.0);
235
+ for (var color_channel = 0u; color_channel < 3u; color_channel += 1u) {
236
+ let weight_offset = (color_channel * 16u + subpixel) * dimensions.channels;
237
+ var correction = 0.0;
238
+ for (var channel = 0u; channel < dimensions.channels; channel += 1u) {
239
+ correction += f32(features[feature_offset + channel]) * f32(weights[weight_offset + channel]);
240
+ }
241
+ residual[color_channel] = correction;
242
+ }
243
+ let output_size = vec2<f32>(f32(dimensions.width * 4u), f32(dimensions.height * 4u));
244
+ let base = textureSampleBaseClampToEdge(source, source_sampler, position.xy / output_size).rgb;
245
+ return vec4<f32>(clamp(base + residual, vec3<f32>(0.0), vec3<f32>(1.0)), 1.0);
246
+ }
247
+ `;
248
+ }
249
+
250
+ export async function createRealtimeVideoUpscaler({
251
+ height,
252
+ model = REALTIME_VIDEO_MODEL,
253
+ width,
254
+ }) {
255
+ if (!realtimeVideoIsEligible({ enabled: true, height, kind: "video", width })) return null;
256
+ const result = await requestBestWebGpuAdapter(globalThis.navigator?.gpu);
257
+ if (!result.adapter?.features?.has("shader-f16")) return null;
258
+ const device = await result.adapter.requestDevice({ requiredFeatures: ["shader-f16"] });
259
+ const contextCanvas = document.createElement("canvas");
260
+ contextCanvas.width = width * model.scale;
261
+ contextCanvas.height = height * model.scale;
262
+ const context = contextCanvas.getContext("webgpu");
263
+ if (!context) {
264
+ device.destroy?.();
265
+ return null;
266
+ }
267
+ const format = navigator.gpu.getPreferredCanvasFormat();
268
+ context.configure({ device, format, alphaMode: "opaque" });
269
+ const payload = await loadModel(model);
270
+ const workWidth = width / 2;
271
+ const workHeight = height / 2;
272
+ const channels = model.channels;
273
+ const featureBytes = workWidth * workHeight * channels * 2;
274
+ const featureUsage = GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST;
275
+ const first = device.createBuffer({ size: featureBytes, usage: featureUsage });
276
+ const second = device.createBuffer({ size: featureBytes, usage: featureUsage });
277
+ const dimensions = mappedBuffer(
278
+ device,
279
+ new Uint32Array([workWidth, workHeight, channels, 0]),
280
+ GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
281
+ );
282
+ const inputWeights = modelBuffer(device, payload, 0, 288);
283
+ const blockWeights = [
284
+ modelBuffer(device, payload, 288, 5184),
285
+ modelBuffer(device, payload, 5472, 5184),
286
+ modelBuffer(device, payload, 10656, 5184),
287
+ ];
288
+ const outputWeights = modelBuffer(device, payload, 15840, 1152);
289
+ const [inputModule, convolutionModule, outputModule] = await Promise.all([
290
+ compiledModule(device, inputShader()),
291
+ compiledModule(device, fusedShader(channels)),
292
+ compiledModule(device, outputShader()),
293
+ ]);
294
+ const inputPipeline = await device.createComputePipelineAsync({
295
+ layout: "auto",
296
+ compute: { module: inputModule, entryPoint: "main" },
297
+ });
298
+ const convolutionPipeline = await device.createComputePipelineAsync({
299
+ layout: "auto",
300
+ compute: { module: convolutionModule, entryPoint: "main" },
301
+ });
302
+ const outputPipeline = await device.createRenderPipelineAsync({
303
+ layout: "auto",
304
+ vertex: { module: outputModule, entryPoint: "vertex_main" },
305
+ fragment: { module: outputModule, entryPoint: "fragment_main", targets: [{ format }] },
306
+ primitive: { topology: "triangle-list" },
307
+ });
308
+ const sampler = device.createSampler({ magFilter: "linear", minFilter: "linear" });
309
+ const convolutionGroups = blockWeights.map((weights, stage) => {
310
+ const source = stage & 1 ? second : first;
311
+ const target = stage & 1 ? first : second;
312
+ return device.createBindGroup({
313
+ layout: convolutionPipeline.getBindGroupLayout(0),
314
+ entries: [source, weights, target, dimensions].map((buffer, binding) => ({
315
+ binding,
316
+ resource: { buffer },
317
+ })),
318
+ });
319
+ });
320
+ let busy = false;
321
+ let destroyed = false;
322
+ let ready = false;
323
+ device.lost.then(() => { destroyed = true; }).catch(() => { destroyed = true; });
324
+
325
+ return {
326
+ backend: "webgpu-f16",
327
+ canvas: contextCanvas,
328
+ get ready() { return ready; },
329
+ destroy() {
330
+ destroyed = true;
331
+ for (const buffer of [first, second, dimensions, inputWeights, outputWeights, ...blockWeights]) {
332
+ buffer.destroy?.();
333
+ }
334
+ device.destroy?.();
335
+ },
336
+ async enhance(source) {
337
+ if (busy || destroyed || !source) return ready ? contextCanvas : null;
338
+ busy = true;
339
+ try {
340
+ const external = device.importExternalTexture({ source });
341
+ const inputGroup = device.createBindGroup({
342
+ layout: inputPipeline.getBindGroupLayout(0),
343
+ entries: [
344
+ { binding: 0, resource: external },
345
+ { binding: 1, resource: sampler },
346
+ { binding: 2, resource: { buffer: first } },
347
+ { binding: 3, resource: { buffer: inputWeights } },
348
+ { binding: 4, resource: { buffer: dimensions } },
349
+ ],
350
+ });
351
+ const outputGroup = device.createBindGroup({
352
+ layout: outputPipeline.getBindGroupLayout(0),
353
+ entries: [
354
+ { binding: 0, resource: external },
355
+ { binding: 1, resource: sampler },
356
+ { binding: 2, resource: { buffer: second } },
357
+ { binding: 3, resource: { buffer: outputWeights } },
358
+ { binding: 4, resource: { buffer: dimensions } },
359
+ ],
360
+ });
361
+ const command = device.createCommandEncoder();
362
+ const compute = command.beginComputePass();
363
+ compute.setPipeline(inputPipeline);
364
+ compute.setBindGroup(0, inputGroup);
365
+ compute.dispatchWorkgroups(Math.ceil(workWidth / 8), Math.ceil(workHeight / 8));
366
+ compute.setPipeline(convolutionPipeline);
367
+ for (const group of convolutionGroups) {
368
+ compute.setBindGroup(0, group);
369
+ compute.dispatchWorkgroups(Math.ceil(workWidth / 8), Math.ceil(workHeight / 8));
370
+ }
371
+ compute.end();
372
+ const render = command.beginRenderPass({
373
+ colorAttachments: [{
374
+ clearValue: { r: 0, g: 0, b: 0, a: 1 },
375
+ loadOp: "clear",
376
+ storeOp: "store",
377
+ view: context.getCurrentTexture().createView(),
378
+ }],
379
+ });
380
+ render.setPipeline(outputPipeline);
381
+ render.setBindGroup(0, outputGroup);
382
+ render.draw(3);
383
+ render.end();
384
+ device.queue.submit([command.finish()]);
385
+ await device.queue.onSubmittedWorkDone();
386
+ ready = true;
387
+ return contextCanvas;
388
+ } finally {
389
+ busy = false;
390
+ }
391
+ },
392
+ };
393
+ }
browser/runtimeCapabilities.js ADDED
@@ -0,0 +1,106 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ const ADAPTER_ATTEMPTS = [
2
+ { mode: "default" },
3
+ { mode: "high-performance", options: { powerPreference: "high-performance" } },
4
+ { mode: "low-power", options: { powerPreference: "low-power" } },
5
+ { mode: "compatibility", options: { featureLevel: "compatibility" } },
6
+ {
7
+ mode: "compatibility-low-power",
8
+ options: { featureLevel: "compatibility", powerPreference: "low-power" },
9
+ },
10
+ {
11
+ mode: "software-fallback",
12
+ options: { featureLevel: "compatibility", forceFallbackAdapter: true },
13
+ },
14
+ ];
15
+
16
+ let computeCapabilityPromise;
17
+
18
+ function errorMessage(error) {
19
+ return error instanceof Error ? error.message : String(error);
20
+ }
21
+
22
+ export async function requestBestWebGpuAdapter(gpu) {
23
+ if (typeof gpu?.requestAdapter !== "function") {
24
+ return { adapter: null, attempts: [], mode: null, reason: "api-unavailable" };
25
+ }
26
+
27
+ const attempts = [];
28
+ for (const attempt of ADAPTER_ATTEMPTS) {
29
+ let adapter;
30
+ try {
31
+ adapter = await gpu.requestAdapter(attempt.options);
32
+ if (!adapter) {
33
+ attempts.push({ mode: attempt.mode, result: "no-adapter" });
34
+ continue;
35
+ }
36
+ const device = await adapter.requestDevice();
37
+ device.destroy?.();
38
+ const readyAdapter = await gpu.requestAdapter(attempt.options);
39
+ if (!readyAdapter) {
40
+ attempts.push({ mode: attempt.mode, result: "adapter-refresh-failed" });
41
+ continue;
42
+ }
43
+ attempts.push({ mode: attempt.mode, result: "ready" });
44
+ return { adapter: readyAdapter, attempts, mode: attempt.mode, reason: "ready" };
45
+ } catch (error) {
46
+ attempts.push({
47
+ message: errorMessage(error),
48
+ mode: attempt.mode,
49
+ result: adapter ? "device-failed" : "adapter-failed",
50
+ });
51
+ }
52
+ }
53
+
54
+ const reason = attempts.some((attempt) => attempt.result === "device-failed")
55
+ ? "device-unavailable"
56
+ : "adapter-unavailable";
57
+ return { adapter: null, attempts, mode: null, reason };
58
+ }
59
+
60
+ export async function probeComputeCapability({
61
+ gpu = globalThis.navigator?.gpu,
62
+ secureContext = globalThis.isSecureContext,
63
+ } = {}) {
64
+ if (secureContext === false) {
65
+ return {
66
+ backend: "wasm",
67
+ label: "CPU optimized",
68
+ mode: null,
69
+ reason: "insecure-context",
70
+ webGpu: false,
71
+ };
72
+ }
73
+
74
+ const result = await requestBestWebGpuAdapter(gpu);
75
+ return {
76
+ backend: result.adapter ? "webgpu" : "wasm",
77
+ label: result.adapter ? "GPU accelerated" : "CPU optimized",
78
+ mode: result.mode,
79
+ reason: result.reason,
80
+ webGpu: Boolean(result.adapter),
81
+ };
82
+ }
83
+
84
+ export function getComputeCapability() {
85
+ if (!computeCapabilityPromise) {
86
+ computeCapabilityPromise = probeComputeCapability().catch((error) => {
87
+ console.warn("Local GPU probe failed; continuing with browser CPU.", error);
88
+ return {
89
+ backend: "wasm",
90
+ label: "CPU optimized",
91
+ mode: null,
92
+ reason: "probe-error",
93
+ webGpu: false,
94
+ };
95
+ });
96
+ }
97
+ return computeCapabilityPromise;
98
+ }
99
+
100
+ export async function hasWebGpuAdapter() {
101
+ return (await getComputeCapability()).webGpu;
102
+ }
103
+
104
+ export function resetComputeCapability() {
105
+ computeCapabilityPromise = undefined;
106
+ }
config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "RealtimeLumaStudent"
4
+ ],
5
+ "model_type": "clark_realtime_luma_x2",
6
+ "channels": 12,
7
+ "blocks": 3,
8
+ "color_mode": "rgb",
9
+ "scale": 2,
10
+ "parameters": 8496,
11
+ "browser_format": "backdropkit-realtime-luma-student-f16-v1",
12
+ "torch_dtype": "float32"
13
+ }
examples/product-demo-1-before-after.png ADDED

Git LFS Details

  • SHA256: 60854a02ceb57ef5e47a93e337f60a5ac32290dad13d00672e5765f81d98e44b
  • Pointer size: 131 Bytes
  • Size of remote file: 244 kB
examples/product-demo-2-before-after.png ADDED

Git LFS Details

  • SHA256: d8cfc984b9ae2dc92599e1016f9d349866853aac2bc1497784f9f83fcfe990af
  • Pointer size: 131 Bytes
  • Size of remote file: 158 kB
examples/product-demo-3-before-after.png ADDED

Git LFS Details

  • SHA256: 67db657103068aa4432c55b7a732078a77c468845fa58e592f7b141ae530429b
  • Pointer size: 131 Bytes
  • Size of remote file: 201 kB
examples/product-demo-before-after.png ADDED

Git LFS Details

  • SHA256: 17635917575305127c4bdb2c70279c729c1668e818c13c6f20aec40f3735e6d4
  • Pointer size: 131 Bytes
  • Size of remote file: 602 kB
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:40d0c8b9f0c28095e06261c56a4c464e0b93c55b845f2470de175e8fa710b6df
3
+ size 34712
realtime_student.py ADDED
@@ -0,0 +1,170 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Training and export contract for the BackdropKit realtime x2 video student."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from pathlib import Path
7
+
8
+ import numpy as np
9
+ import torch
10
+ from torch import nn
11
+ from torch.nn import functional as F
12
+
13
+
14
+ @dataclass(frozen=True)
15
+ class RealtimeStudentConfig:
16
+ channels: int = 12
17
+ blocks: int = 3
18
+ residual_scale: float = 1.0
19
+ color_mode: str = "luma"
20
+
21
+ def validate(self) -> None:
22
+ if self.channels < 4:
23
+ raise ValueError("channels must be at least four")
24
+ if self.blocks < 1:
25
+ raise ValueError("blocks must be positive")
26
+ if self.color_mode not in {"luma", "rgb"}:
27
+ raise ValueError("color_mode must be luma or rgb")
28
+
29
+
30
+ class ResidualBlock(nn.Module):
31
+ """Two bias-free convolutions matching one fused WGSL dispatch."""
32
+
33
+ def __init__(self, channels: int) -> None:
34
+ super().__init__()
35
+ self.first = nn.Conv2d(channels, channels, 3, padding=1, bias=False)
36
+ self.second = nn.Conv2d(channels, channels, 3, padding=1, bias=False)
37
+
38
+ def forward(self, values: torch.Tensor) -> torch.Tensor:
39
+ hidden = F.relu(self.first(values))
40
+ return F.relu(self.second(hidden) + values)
41
+
42
+
43
+ class RealtimeLumaStudent(nn.Module):
44
+ """A roughly 8K-parameter luma residual model for browser 1080p to 4K."""
45
+
46
+ def __init__(self, config: RealtimeStudentConfig | None = None) -> None:
47
+ super().__init__()
48
+ self.config = config or RealtimeStudentConfig()
49
+ self.config.validate()
50
+ channels = self.config.channels
51
+ input_channels = 4 if self.config.color_mode == "luma" else 12
52
+ output_channels = 16 if self.config.color_mode == "luma" else 48
53
+ self.input_projection = nn.Conv2d(input_channels, channels, 1, bias=False)
54
+ self.blocks = nn.ModuleList(
55
+ ResidualBlock(channels) for _ in range(self.config.blocks)
56
+ )
57
+ self.output_projection = nn.Conv2d(channels, output_channels, 1, bias=False)
58
+ nn.init.zeros_(self.output_projection.weight)
59
+
60
+ @staticmethod
61
+ def luma(values: torch.Tensor) -> torch.Tensor:
62
+ coefficients = values.new_tensor((0.2126, 0.7152, 0.0722)).view(1, 3, 1, 1)
63
+ return (values * coefficients).sum(dim=1, keepdim=True)
64
+
65
+ def residual(self, low_resolution: torch.Tensor) -> torch.Tensor:
66
+ if low_resolution.ndim != 4 or low_resolution.shape[1] != 3:
67
+ raise ValueError("expected NCHW RGB input")
68
+ if low_resolution.shape[-2] % 2 or low_resolution.shape[-1] % 2:
69
+ raise ValueError("input height and width must be even")
70
+ inputs = self.luma(low_resolution) if self.config.color_mode == "luma" else low_resolution
71
+ features = F.pixel_unshuffle(inputs, 2)
72
+ features = F.relu(self.input_projection(features))
73
+ for block in self.blocks:
74
+ features = block(features)
75
+ return F.pixel_shuffle(self.output_projection(features), 4)
76
+
77
+ def forward(self, low_resolution: torch.Tensor) -> torch.Tensor:
78
+ baseline = F.interpolate(
79
+ low_resolution,
80
+ scale_factor=2,
81
+ mode="bilinear",
82
+ align_corners=False,
83
+ )
84
+ correction = self.residual(low_resolution) * self.config.residual_scale
85
+ return (baseline + correction).clamp(0.0, 1.0)
86
+
87
+ def parameter_count(self) -> int:
88
+ return sum(parameter.numel() for parameter in self.parameters())
89
+
90
+
91
+ def wgsl_weight_arrays(model: RealtimeLumaStudent) -> dict[str, np.ndarray]:
92
+ """Return FP16 arrays in the exact storage order used by the fused shaders."""
93
+
94
+ arrays: dict[str, np.ndarray] = {}
95
+ arrays["input_projection"] = (
96
+ model.input_projection.weight.detach().cpu().numpy().reshape(
97
+ model.config.channels,
98
+ model.input_projection.in_channels,
99
+ ).astype(np.float16)
100
+ )
101
+ for index, block in enumerate(model.blocks):
102
+ first = block.first.weight.detach().cpu().permute(0, 2, 3, 1).numpy()
103
+ second = block.second.weight.detach().cpu().permute(0, 2, 3, 1).numpy()
104
+ arrays[f"block_{index}"] = np.concatenate(
105
+ (first.reshape(-1), second.reshape(-1))
106
+ ).astype(np.float16)
107
+ arrays["output_projection"] = (
108
+ model.output_projection.weight.detach().cpu().numpy().reshape(
109
+ model.output_projection.out_channels,
110
+ model.config.channels,
111
+ ).astype(np.float16)
112
+ )
113
+ return arrays
114
+
115
+
116
+ def export_wgsl_weights(model: RealtimeLumaStudent, output: Path) -> dict:
117
+ """Write the tiny FP16 payload and a stable offset ledger."""
118
+
119
+ arrays = wgsl_weight_arrays(model)
120
+ output.parent.mkdir(parents=True, exist_ok=True)
121
+ offset = 0
122
+ ledger = {}
123
+ with output.open("wb") as destination:
124
+ for name, values in arrays.items():
125
+ payload = values.tobytes(order="C")
126
+ destination.write(payload)
127
+ ledger[name] = {
128
+ "offset": offset,
129
+ "bytes": len(payload),
130
+ "elements": values.size,
131
+ }
132
+ offset += len(payload)
133
+ return {
134
+ "format": "backdropkit-realtime-luma-student-f16-v1",
135
+ "channels": model.config.channels,
136
+ "blocks": model.config.blocks,
137
+ "residual_scale": model.config.residual_scale,
138
+ "color_mode": model.config.color_mode,
139
+ "parameters": model.parameter_count(),
140
+ "bytes": offset,
141
+ "tensors": ledger,
142
+ }
143
+
144
+
145
+ def initialize_rgb_from_luma(
146
+ rgb: RealtimeLumaStudent,
147
+ luma_state: dict[str, torch.Tensor],
148
+ ) -> None:
149
+ """Make an RGB student initially reproduce a trained luma student exactly."""
150
+
151
+ if rgb.config.color_mode != "rgb":
152
+ raise ValueError("target student must use RGB residuals")
153
+ rgb_state = rgb.state_dict()
154
+ for name, value in luma_state.items():
155
+ if name.startswith("blocks."):
156
+ rgb_state[name].copy_(value)
157
+ coefficients = luma_state["input_projection.weight"].new_tensor(
158
+ (0.2126, 0.7152, 0.0722)
159
+ )
160
+ source_input = luma_state["input_projection.weight"]
161
+ for channel, coefficient in enumerate(coefficients):
162
+ start = channel * 4
163
+ rgb_state["input_projection.weight"][:, start:start + 4].copy_(
164
+ source_input * coefficient
165
+ )
166
+ source_output = luma_state["output_projection.weight"]
167
+ for channel in range(3):
168
+ start = channel * 16
169
+ rgb_state["output_projection.weight"][start:start + 16].copy_(source_output)
170
+ rgb.load_state_dict(rgb_state, strict=True)
technical/example-evaluation.json ADDED
@@ -0,0 +1,70 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "clark-realtime-ui-video-x2-example-evaluation-v1",
3
+ "selection": "fixed video fractions; detail crop chosen only by reference edge density",
4
+ "frames": [
5
+ {
6
+ "example": "product-demo-1-before-after.png",
7
+ "source": "clark-code-ad-direct-vs-claude-20s-16x9.mp4",
8
+ "timestamp_seconds": 4.402200000000001,
9
+ "reference_size": [
10
+ 1280,
11
+ 720
12
+ ],
13
+ "detail_crop": [
14
+ 84,
15
+ 240,
16
+ 444,
17
+ 460
18
+ ],
19
+ "bicubic_psnr_db": 30.1560115814209,
20
+ "model_psnr_db": 30.932720184326172,
21
+ "bicubic_edge_psnr_db": 27.85646629333496,
22
+ "model_edge_psnr_db": 28.66628646850586
23
+ },
24
+ {
25
+ "example": "product-demo-2-before-after.png",
26
+ "source": "clark-code-demo.mp4",
27
+ "timestamp_seconds": 13.93983408,
28
+ "reference_size": [
29
+ 1280,
30
+ 720
31
+ ],
32
+ "detail_crop": [
33
+ 132,
34
+ 0,
35
+ 492,
36
+ 220
37
+ ],
38
+ "bicubic_psnr_db": 33.195716857910156,
39
+ "model_psnr_db": 34.348419189453125,
40
+ "bicubic_edge_psnr_db": 31.850730895996094,
41
+ "model_edge_psnr_db": 32.78166961669922
42
+ },
43
+ {
44
+ "example": "product-demo-3-before-after.png",
45
+ "source": "clark-code-ad-direct-vs-claude-20s-16x9.mp4",
46
+ "timestamp_seconds": 15.807900000000002,
47
+ "reference_size": [
48
+ 1280,
49
+ 720
50
+ ],
51
+ "detail_crop": [
52
+ 468,
53
+ 276,
54
+ 828,
55
+ 496
56
+ ],
57
+ "bicubic_psnr_db": 33.5679931640625,
58
+ "model_psnr_db": 35.42755889892578,
59
+ "bicubic_edge_psnr_db": 31.71966552734375,
60
+ "model_edge_psnr_db": 33.55112838745117
61
+ }
62
+ ],
63
+ "mean": {
64
+ "bicubic_psnr_db": 32.30657386779785,
65
+ "model_psnr_db": 33.569566090901695,
66
+ "model_gain_db": 1.262992223103841,
67
+ "bicubic_edge_psnr_db": 30.47562090555827,
68
+ "model_edge_psnr_db": 31.666361490885418
69
+ }
70
+ }
technical/package-ledger.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "clark-realtime-ui-video-upscaler-x2-package-v1",
3
+ "checkpoint_sha256": "3cbb8a3358bbeaa239c517f1171dbb7706c2d453c961d17a4e6166aefde8c692",
4
+ "artifacts": {
5
+ "README.md": {
6
+ "bytes": 3867,
7
+ "sha256": "8b3831f2d9151b8f1bb97850b82ba061bb9a3c264f353ed1e4f88ac109446c7e"
8
+ },
9
+ "LICENSE": {
10
+ "bytes": 11348,
11
+ "sha256": "d66ccc11682c7458b19b376e1335e8665cbe412ff71f554bfec25f1791cddfa0"
12
+ },
13
+ "model.safetensors": {
14
+ "bytes": 34712,
15
+ "sha256": "40d0c8b9f0c28095e06261c56a4c464e0b93c55b845f2470de175e8fa710b6df"
16
+ },
17
+ "browser/model.bin": {
18
+ "bytes": 16992,
19
+ "sha256": "c6b4f22d8c38270225b809e97d233eb68d7a48eef17595ac17a60b4540c72dce"
20
+ },
21
+ "browser/model.json": {
22
+ "bytes": 841,
23
+ "sha256": "eba342bf8f54749490fecb9df43f399595c07b291d5188d67bb0f4448d38f09c"
24
+ },
25
+ "browser/realtimeVideoUpscaler.js": {
26
+ "bytes": 16319,
27
+ "sha256": "58dd6544858039b4f8ba7a6cf469a4ed22aadc8588c7483ca7b4af2901ac4742"
28
+ },
29
+ "browser/runtimeCapabilities.js": {
30
+ "bytes": 3145,
31
+ "sha256": "d78fe2ce0f0a4513364599854b363cddae8c1479de17d2cfca006bdd2de931c6"
32
+ },
33
+ "technical/runtime-receipt.json": {
34
+ "bytes": 2152,
35
+ "sha256": "ee1c83e21c13bb6e59264f0159768cb2afd003cdbfaea69c2063629ba6f19580"
36
+ },
37
+ "technical/example-evaluation.json": {
38
+ "bytes": 1875,
39
+ "sha256": "6060c4c1ba7668adb5173eb8bade52f2221b1f2d4733fd1d16bdbef5d35e2722"
40
+ }
41
+ }
42
+ }
technical/runtime-receipt.json ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "backdropkit-realtime-1080p-to-4k-receipt-v2",
3
+ "measured_at": "2026-08-01",
4
+ "device": "Apple M5",
5
+ "browser": "Chrome 150 WebGPU",
6
+ "input": { "width": 1920, "height": 1080, "nominal_fps": 60 },
7
+ "output": { "width": 3840, "height": 2160 },
8
+ "model": {
9
+ "profile": "rgb-ui-video-x2",
10
+ "channels": 12,
11
+ "blocks": 3,
12
+ "parameters": 8496,
13
+ "fp16_bytes": 16992,
14
+ "sha256": "c6b4f22d8c38270225b809e97d233eb68d7a48eef17595ac17a60b4540c72dce",
15
+ "gmac_per_frame": 4.4043264,
16
+ "resident_feature_bytes": 24883200
17
+ },
18
+ "kernel": {
19
+ "name": "RGB pixel-unshuffle plus three two-convolution workgroup-fused residual blocks",
20
+ "tile": [8, 8],
21
+ "precision": "fp16",
22
+ "measured_frames": 120,
23
+ "warmup_frames": 15,
24
+ "median_ms": 8.9,
25
+ "p95_ms": 10.6,
26
+ "maximum_ms": 12.3,
27
+ "frames_per_second_from_median": 112.36
28
+ },
29
+ "trained_end_to_end_with_h264": {
30
+ "measured_frames": 120,
31
+ "warmup_frames": 15,
32
+ "median_ms": 9.4,
33
+ "p95_ms": 11.0,
34
+ "maximum_ms": 11.5,
35
+ "encoded_chunks": 120,
36
+ "encoder_error": null,
37
+ "maximum_encode_queue": 2,
38
+ "remaining_encode_queue": 0,
39
+ "median_encode_latency_ms": 16.7,
40
+ "p95_encode_latency_ms": 28.1
41
+ },
42
+ "browser_cadence_observation": {
43
+ "note": "Compute and encoder latency have 60 fps headroom, but this interactive browser run skipped callbacks; it is not a claim of sustained 60 displayed fps on every browser.",
44
+ "kernel_only_presented_frames": 178,
45
+ "kernel_only_busy_callbacks": 44,
46
+ "encoded_presented_frames": 184,
47
+ "encoded_busy_callbacks": 49
48
+ },
49
+ "training": {
50
+ "steps": 5000,
51
+ "batch_size": 16,
52
+ "training_images": 206232,
53
+ "held_out_images": 268,
54
+ "baseline_product_weighted_ground_truth_psnr_db": 28.788123630892997,
55
+ "final_product_weighted_ground_truth_psnr_db": 29.243351974323687,
56
+ "ground_truth_gain_db": 0.4552283434306903,
57
+ "final_product_weighted_teacher_psnr_db": 32.163547360195835
58
+ },
59
+ "promotion_status": "Default local neural path for eligible BackdropKit video on shader-f16 WebGPU; high-quality browser scaling remains the automatic fallback elsewhere."
60
+ }