HyperLux

Overview over HyperLux features. Each section building on the previous one. The first run downloads scenes and JIT-compiles eight renderer configurations.

%pip install -q "hyperlux[examples]" torch matplotlib

Show imports and helpers

Hide imports and helpers

import os
import time
from dataclasses import replace

import conta
import imageio.v2 as imageio
import matplotlib.pyplot as plt
import numpy as np
import torch
from IPython.display import Video
from matplotlib_inline.backend_inline import set_matplotlib_formats

import hyperlux
from hyperlux.camera import CameraSpec, RGB, Depth, Segmentation, Normals, Flow, FlowStatus, Probes

hyperlux.set_verbosity("warning")
set_matplotlib_formats("jpeg", pil_kwargs={"quality": 90, "optimize": True})


def show(images, titles=(), columns=4, size=3, layout=None, full_resolution=False):
    rows = -(-len(images) // columns)
    figure, axes = plt.subplots(rows, columns, figsize=(size * columns, size * rows), dpi=150, squeeze=False, layout=layout)
    for axis, image, title in zip(axes.flat, images, list(titles) + [""] * len(images)):
        axis.imshow(image, interpolation="nearest")
        axis.set_title(title)
    for axis in axes.flat:
        axis.axis("off")
    if full_resolution:
        figure.canvas.draw()
        scale = max(max(image.shape[1] / axis.bbox.width, image.shape[0] / axis.bbox.height)
                    for axis, image in zip(axes.flat, images))
        figure.set_dpi(np.ceil(figure.dpi * scale))
    plt.show()


def video(frames, path):
    with imageio.get_writer(path, fps=30, codec="libx264") as writer:
        for frame in frames:
            writer.append_data(np.asarray(frame)[..., :3])
    return Video(path, embed=True, width=768, html_attributes='controls loop style="max-width: 100%; height: auto;"')

Scene, camera, frame

scene = hyperlux.Scene()
scene.add_object(hyperlux.primitives.box(size=(6, 6, 0.1), color=(0.5, 0.5, 0.55)), hyperlux.make_transform(position=(0, 0, -0.55)))
scene.add_object(hyperlux.primitives.box(size=(1, 1, 1), color=(0.9, 0.35, 0.1)), hyperlux.make_transform(orientation_wxyz=(0.92, 0, 0, 0.38)))
scene.add_light(hyperlux.SceneLight.point((2, 1, 3), (4, 4, 4), radius=0.3))
renderer = hyperlux.HyperLux(width=1024, height=1024)
renderer.init(scene)
renderer.set_cameras(renderer.camera(position=(3, 1.5, 1.5), look_at=(0, 0, 0), fov=50))
renderer.render()
show([renderer.outputs[RGB()].read()[0]], columns=1, size=8)

print(renderer.backend, renderer.config.key())
print(renderer.config)
studio = hyperlux.HyperLux(config=renderer.config)

optix bb98de75c1536dd3
Config(width=1024, height=1024, cameras=(CameraSpec(name='camera', outputs=(RGB(format='rgba8'),)),), batch_size=1, num_probes=1, fidelity='veryhigh', motion_blur_samples=1, anti_aliasing_grid=2, num_overlays=0, max_overlay_instances=0, max_overlays_per_camera=0, dynamic_motion_blur=False)

Materials, lights, environment

quad = np.array([[0, -0.4, -0.4], [0, 0.4, -0.4], [0, 0.4, 0.4], [0, -0.4, 0.4]], np.float32)
faces = np.array([[0, 1, 2], [0, 2, 3]], np.int32)
materials = hyperlux.Scene()
materials.add_object(hyperlux.primitives.box(size=(8, 8, 0.1), color=(0.5, 0.5, 0.55)), hyperlux.make_transform(position=(0, 0, -1.5)))
for row, roughness in enumerate((0.05, 0.4, 1.0)):
    for column, metallic in enumerate((0.0, 0.5, 1.0)):
        swatch = hyperlux.Object(f"metallic {metallic} roughness {roughness}")
        swatch.add_mesh(hyperlux.Mesh(quad, faces, color=(0.9, 0.5, 0.3), normals=np.tile(np.float32([1, 0, 0]), (4, 1)), tex_coords=quad[:, 1:] + 0.5, metallic=metallic, roughness=roughness))
        materials.add_object(swatch, hyperlux.make_transform(position=(0, 1 - column, 0.9 - 0.9 * row)))
glow = hyperlux.Object("emissive")
glow.add_mesh(hyperlux.Mesh(quad * 0.6, faces, color=(0, 0, 0), emissive=(0.2, 1.5, 3)))
glow.add_light(hyperlux.SceneLight.point((0.3, 0, 0), (0.1, 0.5, 1), attenuation=(1, 0, 1)))
materials.add_object(glow, hyperlux.make_transform(position=(0, -2, 0)))
materials.add_light(hyperlux.SceneLight.directional((0.8, 0.2, 0.57), (0.8, 0.8, 0.75), radius=0.05))
materials.add_light(hyperlux.SceneLight.spot((3, -2, 2), (-0.73, 0.49, -0.49), (2.5, 2.5, 2.5), cos_inner_cone=0.95, cos_outer_cone=0.85, radius=0.2))
materials.set_environment(ambient=(0.15, 0.15, 0.2), background=(0.02, 0.03, 0.05))
studio.init(materials)
studio.set_cameras(studio.camera(position=(4, 0, 0.3), look_at=(0, 0, 0), fov=60))
studio.render()
show([studio.outputs[RGB()].read()[0]], columns=1)

Fidelity tiers

tiers = []
for fidelity in hyperlux.FIDELITY:
    tier = hyperlux.HyperLux(width=256, height=256, fidelity=fidelity)
    tier.init(materials)
    tier.set_cameras(studio.camera(position=(4, 0, 0.3), look_at=(0, 0, 0), fov=60))
    tier.render()
    tiers.append(tier.outputs[RGB()].read()[0])
show(tiers, hyperlux.FIDELITY)

Anti-aliasing and motion blur

smooth = hyperlux.HyperLux(width=256, height=256, motion_blur_samples=8)
smooth.init(scene)
camera_open, camera_close = (renderer.camera(position=(3, y, 1.5), look_at=(0, 0, 0), fov=50) for y in (1.5, 1.9))
aliased = hyperlux.HyperLux(width=256, height=256, anti_aliasing_grid=1)
aliased.init(scene)
aliased.set_cameras(camera_open)
aliased.render()
smooth.set_cameras(camera_open)
smooth.render()
still = smooth.outputs[RGB()].read()[0]
smooth.set_motion_blur_cameras(camera_open, camera_close)
smooth.render_launch()
smooth.render_sync()
crop = np.s_[72:136, 64:128]
show([aliased.outputs[RGB()].read()[0][crop], still[crop], smooth.outputs[RGB()].read()[0][crop]], ("aliased", "2x2 anti-aliasing", "+ 8-sample motion blur"), columns=3)

A real scene, a camera rig

dataset = hyperlux.datasets.procthor.Train200()
house = hyperlux.load_scene(dataset["ProcTHOR-Train-1"].reference, preserve_instances=True)
rig = hyperlux.HyperLux(width=2048, height=2048, batch_size=4, num_probes=32, cameras=(
    CameraSpec("front", (RGB(), Depth(), Segmentation(), Segmentation(labels="semantic"), Normals(), Flow(direction="forward"), FlowStatus(direction="forward"), Probes())),
    CameraSpec("preview", (RGB(),))))
rig.init(house)
P = np.array([-3.92, -5.67, 1.2])
yaws = np.deg2rad([0, 90, 180, 270])
cameras = np.stack([hyperlux.make_camera(P, P + (np.cos(yaw), np.sin(yaw), 0), fov=90, aspect=rig.aspect) for yaw in yaws])
rig.set_cameras(np.repeat(cameras, 2, axis=0))
rig.render()
print(rig.outputs[Depth()].layout)
print(rig.outputs[RGB()].layout.cameras)
show(rig.outputs[RGB()].read()[::2], [f"front, yaw {int(np.degrees(yaw))}°" for yaw in yaws], columns=2, layout="compressed", full_resolution=True)

OutputLayout(shape=(4, 2048, 2048), dtype='float32', axes=('camera', 'height', 'width'), cameras=((0, 'front'), (1, 'front'), (2, 'front'), (3, 'front')), device=(2, 0), block_count=1)
((0, 'front'), (0, 'preview'), (1, 'front'), (1, 'preview'), (2, 'front'), (2, 'preview'), (3, 'front'), (3, 'preview'))

Every output

rig.advance_flow()
cameras = np.repeat(np.stack([hyperlux.make_camera(P + (0.05, 0, 0), P + (np.cos(yaw + 0.02), np.sin(yaw + 0.02), 0), fov=90, aspect=rig.aspect) for yaw in yaws]), 2, axis=0)
rig.set_cameras(cameras)
rig.render()
view = {descriptor: rig.outputs[descriptor].read()[0] for descriptor in rig.outputs if descriptor != Probes()}
valid = view[FlowStatus(direction="forward")] == hyperlux.FlowStatus.VALID
show([np.where(np.isfinite(view[Depth()]), view[Depth()], np.nan), view[Normals()] * 0.5 + 0.5,
      np.ma.masked_equal(view[Segmentation()], 0xFFFFFFFF) % 20, np.ma.masked_equal(view[Segmentation(labels="semantic")], 0xFFFFFFFF) % 20,
      np.ma.masked_where(~valid, np.linalg.norm(view[Flow(direction="forward")], axis=-1))],
     ("depth", "normals", "instances", "classes", "forward flow"), columns=2, size=3.6, layout="compressed", full_resolution=True)

Labels

classes, instances = view[Segmentation(labels="semantic")], view[Segmentation()]
print(house.segmentation_taxonomy, {house.segmentation_classes[int(label)]: int((classes == label).sum()) for label in np.unique(classes) if label != 0xFFFFFFFF})
center = house.instance_object(int(instances[rig.height // 2, rig.width // 2]))
print("center pixel:", house.object_name(center))
coarse = {label: {"Undefined": 0, "Wall": 1, "Floor": 2, "Ceiling": 2}.get(name, 3) for label, name in house.segmentation_classes.items()}
house.remap_segmentation_classes(coarse, taxonomy="coarse", classes={0: "Undefined", 1: "Wall", 2: "Floor/Ceiling", 3: "Furniture", 4: "Target"})
house.set_object_segmentation_class(center, 4)
rig = hyperlux.HyperLux(config=rig.config)
rig.init(house)
rig.set_cameras(cameras)
rig.render()
with plt.rc_context({"axes.titlesize": 4}):
    show([np.ma.masked_equal(rig.outputs[Segmentation(labels="semantic")].read()[0], 0xFFFFFFFF)], ["\n".join(f"{label}: {name}" for label, name in house.segmentation_classes.items())], columns=1, full_resolution=True)

ai2thor-hab {'Undefined': 1275591, 'Bowl': 2777, 'Chair': 71522, 'Plate': 2052, 'DiningTable': 75684, 'Television': 61259, 'Painting': 80931, 'Laptop': 15854, 'RemoteControl': 260, 'Newspaper': 1998, 'CreditCard': 83, 'Book': 3046, 'Pen': 70, 'Pencil': 109, 'Watch': 75, 'Sofa': 139332, 'Pillow': 4691, 'DeskLamp': 7193, 'TVStand': 46876, 'Floor': 1282724, 'Wall': 1122177}
center pixel: Wall#wall|5|0.00|3.51|0.00|8.77

Probes and free space

rig.generate_probe_directions()
rig.probe()
distance, hit = rig.outputs[Probes()].distance.read(), rig.outputs[Probes()].hit.read()
free_space = hyperlux.FreeSpace(house, parameters=hyperlux.FreeSpaceParameters(min_clearance=0.6))
points = free_space.sample(4, seed=0)

Assets and overlays

x500 = conta.resolve({"description": "rigged/x500_optimized_scaled.glb", "hash": "c0fda2143b7a2f0422148aa8f6c4e24db3f8a6e6"})
drone = hyperlux.load_assembly(x500, part_nodes=["body", "prop_0", "prop_1", "prop_2", "prop_3"])
for prop in range(4):
    low, high = drone.object_bounds(prop + 1)
    drone.set_joint(f"prop_{prop}", "revolute", axis=(0, 0, 1), pivot=(low + high) / 2, coordinate=prop)
payload = hyperlux.primitives.box(size=(0.08, 0.08, 0.12), color=(1, 0.8, 0))
payload.name = "payload"
drone.add_part(payload, hyperlux.make_transform(position=(0, 0, -0.15)), parent="body")
drone.set_joint("payload", "prismatic", axis=(0, 0, -1), coordinate=4)
drone.remap_segmentation_classes({0: 3}, taxonomy="coarse", classes=house.segmentation_classes)
print(drone.object_names(), drone.num_coordinates, drone.joint("prop_0"), drone.part_parent(5))
D = P + (0, 0, -0.2)
house.add_assembly(drone, hyperlux.make_transform(position=D + (0.5, 0.9, -0.9)))
pool = hyperlux.AssetPool()
asset = pool.add_assembly(drone)
overlay_config = replace(rig.config, width=1024, height=1024, batch_size=1, num_overlays=1, max_overlay_instances=36, max_overlays_per_camera=1, motion_blur_samples=8, dynamic_motion_blur=True)
del rig
overlay = hyperlux.HyperLux(config=overlay_config)
overlay.init(house, asset_pool=pool)
overlay.attach(0, 0)
placement = overlay.spawn(0, asset, hyperlux.Pose(hyperlux.make_transform(position=D)), segmentation_class=hyperlux.AGENT_SEGMENTATION_CLASS)
overlay.set_cameras(np.repeat(overlay.camera(position=D + (-1.2, -1.2, 0.7), look_at=D, fov=55)[None], 2, axis=0))
overlay.update()
overlay.render()
ids = house.num_instances + placement.overlay * overlay.config.max_overlay_instances + placement.first_slot + np.arange(placement.num_parts)
print(placement, ids)
show([*overlay.outputs[RGB()].read(), np.isin(overlay.outputs[Segmentation()].read()[0], ids), overlay.outputs[Segmentation(labels="semantic")].read()[0] == hyperlux.AGENT_SEGMENTATION_CLASS],
     ("front: overlay attached", "preview: not attached", "overlay instance ids", "AGENT class"), columns=2, layout="compressed")

['body', 'prop_0', 'prop_1', 'prop_2', 'prop_3', 'payload'] 5 ('revolute', array([0., 0., 1.], dtype=float32), array([-0.17878559, -0.17474456,  0.11214785], dtype=float32), 0) 0

OverlayPlacement(overlay=0, first_slot=0, num_parts=6, _generation=0, _renderer=11) [295 296 297 298 299 300]

Articulated motion blur

def state(t):
    base = hyperlux.make_transform(position=D + (0, 0, 0.05 * np.sin(2 * t)), orientation_wxyz=(np.cos(t / 2), 0, 0, np.sin(t / 2)))
    coordinates = [150 * t, -150 * t, 150 * t, -150 * t, 0.1 + 0.1 * np.sin(3 * t)]
    camera = overlay.camera(position=D + (-1.2 * np.cos(t / 3), -1.2 * np.sin(t / 3), 0.7), look_at=D, fov=55)
    return base, coordinates, np.repeat(camera[None], 2, axis=0)


def motion_frames():
    for t in np.arange(0, 4, 1 / 30):
        (base_open, coordinates_open, cameras_open), (base_close, coordinates_close, cameras_close) = state(t), state(t + 0.5 / 30)
        overlay.set_pose(placement, hyperlux.Pose(base_open), hyperlux.Pose(base_close), coordinates=coordinates_open, coordinates_close=coordinates_close)
        overlay.set_motion_blur_cameras(cameras_open, cameras_close)
        overlay.update()
        overlay.render()
        yield overlay.outputs[RGB()].read()[0]


video(motion_frames(), "hyperlux_motion.mp4")

Bulk posing on the GPU

overlay.despawn(placement)
placements = [overlay.spawn(0, asset) for _ in range(5)]
batch = overlay.placement_batch(placements)
device = torch.device("cuda" if overlay.backend == "optix" else "cpu")
stream = torch.cuda.current_stream(device).cuda_stream if device.type == "cuda" else 0
angles = torch.linspace(0, 2 * torch.pi, 6, device=device)[:-1]
rest = torch.eye(4, device=device)[:3].repeat(5, 1, 1)
rest[:, :, 3] = torch.as_tensor(D, dtype=torch.float32, device=device) + 0.5 * torch.stack((angles.cos(), angles.sin(), 0.3 * (2 * angles).sin()), dim=1)
spin = torch.tensor([150, -150, 150, -150, 0], device=device)
overlay.set_cameras(torch.as_tensor(np.repeat(overlay.camera(position=D + (-1.8, -1.4, 0.9), look_at=D, fov=65)[None], 2, axis=0), device=device), stream=stream)


def swarm_frames():
    frame = torch.empty(overlay.outputs[RGB()].layout.shape, dtype=torch.uint8, device=device)
    for i in range(120):
        t = i / 30
        bases = rest.clone()
        bases[:, 2, 3] += 0.1 * (2 * t + angles).sin()
        overlay.set_pose(batch, hyperlux.Pose(bases), coordinates=spin.expand(5, 5) * t, stream=stream)
        overlay.update_launch()
        overlay.render_launch()
        overlay.outputs[RGB()].read(out=frame, stream=stream)
        yield frame[0].cpu().numpy()


start = time.perf_counter()
swarm_video = video(swarm_frames(), "hyperlux_swarm.mp4")
print(f"{120 / (time.perf_counter() - start):.0f} frames/s including encoding on {device}, render stream {overlay.render_stream:#x}")
live = torch.from_dlpack(overlay.outputs[RGB()].blocks[0])
print(live.shape, live.dtype, live.device, torch.from_dlpack(overlay.outputs[RGB()].packed.blocks[0]).dtype)
swarm_video

10 frames/s including encoding on cuda, render stream 0x55083d50
torch.Size([2, 1024, 1024, 4]) torch.uint8 cuda:0 torch.uint32

Scene catalogs

train = hyperlux.datasets.procthor.Train200()
validation = hyperlux.datasets.hssd.Val43()
print(train.name, train.source, train.split, len(train), train[0].id, train[0].reference)
print(validation.name, validation.source, validation.split, len(validation), validation[0].id, validation[0].reference)
entries = (train[0], train[1], train[2], validation[0])
rng = np.random.default_rng(0)
figure, axes = plt.subplots(len(entries), 4, figsize=(12, 3 * len(entries)), dpi=150, squeeze=False)
figure.subplots_adjust(left=0, right=1, bottom=0, top=1, wspace=0, hspace=0)
for entry, row in zip(entries, axes):
    scene = hyperlux.load_scene(entry.reference)
    viewer = hyperlux.HyperLux(config=replace(renderer.config, batch_size=4))
    viewer.init(scene)
    positions = hyperlux.FreeSpace(scene, parameters=hyperlux.FreeSpaceParameters(min_clearance=0.8)).sample(4, seed=0)
    yaws = rng.uniform(-np.pi, np.pi, 4)
    viewer.set_cameras(np.stack([
        viewer.camera(position=position, look_at=position + (np.cos(yaw), np.sin(yaw), 0), fov=90)
        for position, yaw in zip(positions, yaws)
    ]))
    viewer.render()
    for axis, image in zip(row, viewer.outputs[RGB()].read()):
        axis.imshow(image, interpolation="nearest")
        axis.axis("off")
    figure.text(0.03, 0.97, entry.id, transform=row[0].transAxes, va="top", color="white", fontsize=32,
                bbox={"facecolor": "black", "alpha": 0.6, "edgecolor": "none", "pad": 3})
with plt.rc_context({"savefig.pad_inches": 0}):
    plt.show()

procthor_train_200 ai2thor-hab/ProcTHOR train 200 ProcTHOR-Train-0 conta:3ee4771c22b54fa7bb57022ac2eb5a015f387b4d
hssd_val_43 hssd/hssd-hab val 43 107734158_175999998 conta:e5e59b08d09e1192a2a176d05f8006c5d5c21b5f