Performing accurate time synchronization between two devices

You can use lsl to synchronize two Pupil Labs Companion Devices.

Description of the Choreography

We placed two Pupil Invisible Glasses in front of a light source, the eye cameras facing towards the light. The scene cameras were detached from the glasses and the light source was switched off during setup. The Companion Devices were set up such that both would upload the recorded data to the same Workspace.

An lsl relay was started for each of the Pupil Invisible Glasses, and the two lsl Event streams were recorded with the LabRecorder. We did not record the lsl Gaze streams for this example.

After the lsl streams were running and being recording by the LabRecorder, we started recording in the Pupil Invisible Companion App. After this step, the following data series were being recorded:

• Event data from each of the two Pupil Invisible Glasses (two data streams) are streamed and recorded via lsl and the LabRecorder.

• Event data from each pair of Pupil Invisible Glasses is saved locally on the Companion Device during the recording and uploaded to cloud once the recording completed.

• Eye Camera Images from each pair of Pupil Invisible Glasses are saved locally on the Companion Device and uploaded to Pupil Cloud when the recording completed. The 200 Hz Gaze position estimate is computed in Pupil Cloud.

With this setup in place and all recordings running, we switched the light source pointing at the eye cameras on and off four times. This created a simultaneous signal recorded by the Eye Cameras (brightness increases).

After that, we stopped all recordings in the inverse order (first in the companion app, then in the LabRecorder) and stopped the lsl relay.

Comparing NTP and LSL time sync

We used the Start the post-hoc time alignment tool to calculate the time alignments between the two Companion devices and the computer running the LSL relay.

Time Alignment Parameters - Subject 1

{
    "cloud_to_lsl": {
        "intercept": -1666236720.9472754,
        "slope": 0.9999924730866947
    },
    "lsl_to_cloud": {
        "intercept": 1666249262.6610205,
        "slope": 1.0000075269698494
    },
    "info": {
        "model_type": "LinearRegression",
        "version": 1
    }
}

Time Alignment Parameters - Subject 2

{
    "cloud_to_lsl": {
        "intercept": -1666236286.8392515,
        "slope": 0.9999922119433565
    },
    "lsl_to_cloud": {
        "intercept": 1666249263.6828992,
        "slope": 1.0000077881172447
    },
    "info": {
        "model_type": "LinearRegression",
        "version": 1
    }
}

Then we extracted the illuminance for each eye video frame

Illuminance Extraction Script

from pathlib import Path
from typing import Collection

import av
import click
import numpy as np
import pandas as pd
from rich import print
from rich.progress import track
from rich.traceback import install as install_rich_traceback


@click.command()
@click.argument(
    "eye_videos",
    type=click.Path(exists=True, readable=True, dir_okay=False, path_type=Path),
    nargs=-1,
)
def main(eye_videos: Collection[Path]):
    if not eye_videos:
        raise ValueError(
            "Please pass at least one path to an `PI right/left v1 ps*.mp4/mjpeg` file."
        )
    for path in eye_videos:
        extract_illuminance(path)


def extract_illuminance(eye_video_path: Path):
    print(f"Processing {eye_video_path}...")
    time = _load_time(eye_video_path.with_suffix(".time"))
    illuminance = _decode_illuminance(eye_video_path, num_expected_frames=time.shape[0])

    min_len = min(time.shape[0], illuminance.shape[0])
    time = time.iloc[:min_len]
    illuminance = illuminance.iloc[:min_len]

    output_path = eye_video_path.with_suffix(".illuminance.csv")
    print(f"Writing result to {output_path}")
    pd.concat([illuminance, time], axis="columns").to_csv(output_path, index=False)


def _load_time(time_path: Path) -> "pd.Series[int]":
    return pd.Series(np.fromfile(time_path, dtype="<u8"), name="timestamp [ns]")


def _decode_illuminance(video_path: Path, num_expected_frames: int) -> "pd.Series[int]":
    video_format = video_path.suffix[1:]
    with av.open(str(video_path), format=video_format) as container:
        container.streams.video[0].thread_type = "AUTO"
        illuminance = [
            np.array(frame.planes[0], dtype=np.uint8).sum()
            for frame in track(
                container.decode(video=0),
                description="Decoding illuminance",
                total=num_expected_frames,
            )
        ]
    return pd.Series(illuminance, name="illuminance")


if __name__ == "__main__":
    install_rich_traceback()
    main()

Extracted Illuminance - Excerpt Subject 1 - Left Eye Video

illuminance,timestamp [ns]
1930703,1666344684449078006
1931568,1666344684457110006
1937257,1666344684461092006
1930975,1666344684465084006
1931744,1666344684469134006
1935741,1666344684477145006
1931907,1666344684481116006
1931363,1666344684485168006
1935227,1666344684489040006

With the script below, we load the time alignment configurations, apply them on the extracted illuminance timestamps, and plot the illuminance for both recordings over original and aligned timestamps.

Script to Apply and Plot Time Alignment

from itertools import chain
from pathlib import Path

import pandas as pd
import seaborn as sns
from rich import print
from rich.traceback import install

from pupil_labs.lsl_relay.linear_time_model import TimeAlignmentModels

time_domain_key = "Pupil Companion Device"

install()
sns.set(font_scale=1.5)

illuminance_files = sorted(
    Path("./companion_app_exports").glob("subject_*/*.illuminance.csv")
)
print(f"Illuminance files: {sorted(map(str, illuminance_files))}")
companion_device_dfs = {f: pd.read_csv(f) for f in illuminance_files}

for path, df in companion_device_dfs.items():
    df["path"] = path
    df["time"] = pd.to_datetime(df["timestamp [ns]"], unit="ns")
    df["time [s]"] = df["timestamp [ns]"] * 1e-9
    df["subject"] = path.parent.name
    df["eye"] = path.stem.split(" ")[1]

    ill_min = df.illuminance.min()
    ill_max = df.illuminance.max()
    df["illuminance (normalized)"] = (df.illuminance - ill_min) / (ill_max - ill_min)

    df["time domain"] = time_domain_key

lsl_time_dfs = {p: df.copy() for p, df in companion_device_dfs.items()}
for path, df in lsl_time_dfs.items():
    assert (df.path == path).all()
    model_path = path.with_name("time_alignment_parameters.json")
    models = TimeAlignmentModels.read_json(model_path)
    df["time [s]"] = df["timestamp [ns]"] * 1e-9

    df["time [s]"] = models.cloud_to_lsl.predict(df[["time [s]"]].values)
    df["time domain"] = "Lab Streaming Layer"

illuminance_df = pd.concat(
    chain(companion_device_dfs.values(), lsl_time_dfs.values()), ignore_index=True
)


for time_domain in (time_domain_key, "Lab Streaming Layer"):
    for eye in ("left", "right"):
        mask = (illuminance_df["time domain"] == time_domain) & (
            illuminance_df.eye == eye
        )
        illuminance_df.loc[mask, "time [s]"] -= illuminance_df.loc[
            mask, "time [s]"
        ].min()

fg = sns.relplot(
    kind="line",
    data=illuminance_df,
    x="time [s]",
    y="illuminance (normalized)",
    col="eye",
    hue="subject",
    row="time domain",
    aspect=1,
    facet_kws=dict(sharex=False),
)
fg.set_titles(template="{row_var} =\n{row_name}\n—\n{col_var} = {col_name}")
fg.tight_layout()

output_path = Path("illuminance_over_time.png").resolve()
print(f"Saving figure to {output_path}")
fg.savefig(output_path, dpi=120)

The resulting plot demonstrates the improved time sync when using the LSL-aligned timestamps.