Build uploaded using `kernels`.

build/torch-universal/__init__.py

@@ -0,0 +1,11 @@
+import torch
+
+from ._ops import ops
+from .silu_and_mul import _silu_and_mul
+
+
+def silu_and_mul(x: torch.Tensor) -> torch.Tensor:
+    return ops.silu_and_mul(x)
+
+
+__all__ = ["silu_and_mul"]

build/torch-universal/_ops.py

@@ -0,0 +1,8 @@
+import torch
+ops = torch.ops._flattened_build_19700101000000
+
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_flattened_build_19700101000000::{op_name}"

build/torch-universal/flattened_build/__init__.py

@@ -0,0 +1,26 @@
+import ctypes
+import sys
+
+import importlib
+from pathlib import Path
+from types import ModuleType
+
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+
+
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch-universal/silu_and_mul.py

@@ -0,0 +1,35 @@
+import torch
+import torch.nn.functional as F
+
+from ._ops import add_op_namespace_prefix
+
+
+@torch.library.custom_op(add_op_namespace_prefix("silu_and_mul"), mutates_args=())
+def _silu_and_mul(x: torch.Tensor) -> torch.Tensor:
+    d = x.shape[-1] // 2
+    return F.silu(x[..., :d]) * x[..., d:]
+
+
+def backward(ctx, grad_output):
+    x = ctx.saved_tensors[0]
+    d = x.shape[-1] // 2
+    x1, x2 = x[..., :d], x[..., d:]
+    sigmoid_x1 = torch.sigmoid(x1)
+    silu_x1 = F.silu(x1)
+    dsilu_dx1 = sigmoid_x1 + silu_x1 * (1 - sigmoid_x1)
+    dx1 = grad_output * x2 * dsilu_dx1
+    dx2 = grad_output * silu_x1
+    return torch.cat([dx1, dx2], dim=-1)
+
+
+def setup_context(ctx, inputs, output):
+    (x,) = inputs
+    ctx.save_for_backward(x)
+
+
+_silu_and_mul.register_autograd(backward, setup_context=setup_context)
+
+
+@_silu_and_mul.register_fake
+def _(x: torch.Tensor) -> torch.Tensor:
+    return x.new_empty(x.shape[0], x.shape[1] // 2)