Initial Commit

app.py

@@ -0,0 +1,183 @@
+import gradio as gr
+import json
+from datetime import datetime
+import random
+import os
+from huggingface_hub import Repository
+random.seed(1234)
+
+import subprocess
+# Set Git user information
+subprocess.run(["git", "config", "--global", "user.email", "[email protected]"])
+subprocess.run(["git", "config", "--global", "user.name", "yiduohao"])
+
+
+hf_token = os.getenv("HF_TOKEN")
+print("HF Token is none?", hf_token is None)
+
+# Initialize the repository
+DATASET_REPO_URL = "https://huggingface.co/datasets/Scherzando/RIR-Resound-User-Study-Response"
+repo = Repository(
+    local_dir="user_responses",
+    clone_from=DATASET_REPO_URL,
+    use_auth_token=hf_token
+)
+
+def prepare_test_cases():
+    # json_path = "videos/videos.json"
+    json_path = "rir/rir.json"
+    with open(json_path, "r") as f:
+        video_dict = json.load(f)
+    
+    video_ids = list(video_dict.keys())
+    for video_id in video_ids:
+        if random.random() > 0.5:
+            video_list = [video_dict[video_id]['ours'], video_dict[video_id]['baseline']]
+        else: 
+            video_list = [video_dict[video_id]['baseline'], video_dict[video_id]['ours']]
+
+        random.shuffle(video_list)
+
+        video_dict[video_id]['Video 1'] = video_list[0]
+        video_dict[video_id]['Video 2'] = video_list[1]
+        video_dict[video_id]['Ground Truth'] = video_dict[video_id]['groundtruth']
+
+    return video_dict
+
+video_dict = prepare_test_cases()
+video_ids = list(video_dict.keys())  
+random.shuffle(video_ids)
+
+
+questions = [
+    "Between Video 1 and Video 2, which one's audio conveyed changes in audio volume more accurately compared to the Reference?",
+    "Between Video 1 and Video 2, which one's audio made it easier for you to identify the direction of the sound source more accurately?",
+    "Between Video 1 and Video 2, which one's audio do you feel aligns better with the Reference overall?"
+]
+submissions_file = "user_responses/response.jsonl"
+
+def has_already_submitted(user_id):
+    if os.path.exists(submissions_file):
+        with open(submissions_file, "r") as f:
+            for line in f:
+                submission = json.loads(line)
+                if submission.get("u_id") == user_id:
+                    return True
+    return False
+
+# Save responses
+def save_responses(unique_submission, *responses):
+    timestamp = datetime.now().isoformat()
+    info = responses[-1]
+    responses = responses[:-1]
+    unique_id = info["session_id"]
+    user_id = f"{unique_id}"
+
+    # Check for unique submission
+    if unique_submission and has_already_submitted(user_id):
+        return "You have already submitted responses. Thank you for participating!"
+
+    # Initialize the result dictionary
+    result = {
+        "u_id": user_id,
+        "timestamp": timestamp,
+        "responses": []
+    }
+
+    for index in range(len(video_ids)):
+        start_idx = index * len(questions)
+        end_idx = start_idx + len(questions)
+
+        response = responses[start_idx:end_idx]
+        if any(r is None for r in response):
+            return "Please answer all questions before submitting."
+
+        video_id = video_ids[index]
+        pair_response = {
+            video_id: {
+                'distance': video_dict[video_id][response[0]],
+                'direction': video_dict[video_id][response[1]],
+                'overall': video_dict[video_id][response[2]],
+            }
+        }
+        result["responses"].append(pair_response)
+
+    result["responses"] = sorted(result["responses"], key=lambda x: x.keys())
+    
+    # Save response locally and push to Hugging Face Hub
+    with open(submissions_file, "a") as f:
+        f.write(json.dumps(result) + "\n")
+    
+    # Push changes to the Hugging Face dataset repo
+    repo.push_to_hub()
+
+    return "All responses saved! Thank you for participating!"
+
+
+
+
+def create_interface(unique_submission=False):
+    with gr.Blocks() as demo:
+        gr.Markdown("# Human Preference Study: Room Spatial Audio Rendering")
+        gr.Markdown("""
+            Before starting the study, please make sure you are in a quiet environment and wearing headphones, and read the following guidance carefully. 
+            - In this study, you will be presented with pairs of videos **with spatial audio**. 
+            - Each pair consists of a reference spatial audio (marked by **Reference**) and two generated spatial audio (marked by **Video 1** and **Video 2**). 
+            - Please watch and **listen** to each row of videos carefully and answer the three associated questions. 
+            - For each video, the left-hand side is the camera (head) view, and the right-hand side is the corresponding birds-eye view of the room with speaker and head poses.
+            
+            **Binanural Headphones are required!**
+        """)
+
+        # Display video pairs and questions
+        responses = []
+        for index, video_id in enumerate(video_ids):
+            video_gt = video_dict[video_id]['groundtruth']
+            video1 = video_dict[video_id]['Video 1']
+            video2 = video_dict[video_id]['Video 2']
+
+            gr.Markdown(f"### Video Pair {index + 1}")
+            with gr.Row():
+                gr.Video(video_gt, label="Reference")
+                gr.Video(video1, label="Video 1")
+                gr.Video(video2, label="Video 2")
+                # with gr.Column():
+            with gr.Row():
+                responses.append(gr.Radio(["Video 1", "Video 2"], label=questions[0], value=None))
+            with gr.Row():
+                responses.append(gr.Radio(["Video 1", "Video 2"], label=questions[1], value=None))
+            with gr.Row():
+                responses.append(gr.Radio(["Video 1", "Video 2"], label=questions[2], value=None))
+
+            gr.Markdown("---")
+
+        info = gr.JSON(visible=False)
+        demo.load(predict, None, info)
+
+        submit_btn = gr.Button("Submit")
+        result_message = gr.Textbox(label="Message (please only submit once)", interactive=False)
+
+        submit_btn.click(
+            fn=lambda *args: save_responses(unique_submission, *args),
+            inputs=responses+[info],
+            outputs=result_message
+        )
+
+    return demo
+
+def predict(request: gr.Request):
+    headers = request.headers
+    host = request.client.host
+    user_agent = request.headers["user-agent"]
+    session_id = request.session_hash
+    return {
+        "ip": host,
+        "user_agent": user_agent,
+        "headers": headers,
+        "session_id": session_id
+    }
+
+if __name__ == "__main__":    
+    # Launch with unique_submission set based on `--unique` flag
+    demo = create_interface(unique_submission=True)
+    demo.launch(share=True)

rir/baseline/combined_bene_tx_rotation_ele.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f501deb9a53ddaeb90d07275d54906ee0998646676783c18fdf6a79862462b9e
+size 5869131

rir/baseline/combined_bene_vanilla.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4504e141c25fc6f036dccd9ef200db2948f5bab5040f64c327059f56234491d
+size 8269383

rir/baseline/combined_pomaria_tx_moving_vanilla.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:adf3c069d9801c0efbb97ee86d0eb4e7d373b0b9f3da02a453c874a17b9abe41
+size 9600134

rir/baseline/combined_pomaria_tx_rotation_ele.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48173e1a932307dda06cf3050c4233b7ca468c24530e8d5bb8b0bbeb90567ba7
+size 8442679

rir/baseline/combined_pomaria_vanilla.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5d249eef893134935af384de1e877e01fbca724683e736a84e3f4db7471d08d
+size 9534331

rir/baseline/combined_rs_tx_rotation1_ele.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0027e90b39121a8cd26e912383a43c935afea0b45c8fd6d7fdc70bb8229059b
+size 7458436

rir/baseline/combined_rs_tx_rotation2_ele.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:696a1c4657855d127fbd0bddba4f2706579ec2e05fcf9ed8d12763d43503358a
+size 6808873

rir/baseline/combined_rs_vanilla.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1834bc234fef9585b714bf86a0d072bc8a8dafd77b0ee16a24046f6a084ec21a
+size 9761008

rir/groundtruth/combined_bene_gt.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa037d01e53bd5d3d840a130a95398c2b3018ad1558d9d380040f80317237107
+size 8269383

rir/groundtruth/combined_bene_tx_rotation_gt.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b68ca74647aac794e5eafb0a9ce7bdebb618939e05c32a8a3dc630acb29f948
+size 5869131

rir/groundtruth/combined_pomaria_gt.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb3229d91ff91f06670fe1c6fde92a62a90c8961a2690b877156da29d4818fcd
+size 9534331

rir/groundtruth/combined_pomaria_tx_moving_gt.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc7212ba964aa73710a32af715d187b3e657b2a1af8efc534ab684b897a35d97
+size 9600134

rir/groundtruth/combined_pomaria_tx_rotation_gt.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a47a73036d818a7cb10e7e4e59abbdc1dd913bd673e2d571c2e4585c243c65d9
+size 8442679

rir/groundtruth/combined_rs_gt.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b82a5d68d2973dc1195d44616182b1f047aed72e288e5f968805fa35897354ce
+size 9761008

rir/groundtruth/combined_rs_tx_rotation1_gt.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8ba7802becbdff0b2a4ce57bcbdcdf6a2597a4ed24d121ee7484e4613dbcb2f
+size 7458436

rir/groundtruth/combined_rs_tx_rotation2_gt.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:089422fa95fda67f7a5eef34bc1895f20b09f9934f6185894cbcf55a214235c3
+size 6808873

rir/ours/combined_bene_ele.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f6cf778948ce2fcb52153006b20ab288ae134ad790a507ba835167ed3ba6575
+size 8269383

rir/ours/combined_bene_tx_rotation_ssl.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c941f16ff17fe7bbd3bea73620fac2b0e435bdee5dd27670ba1838fd5843b825
+size 5869131

rir/ours/combined_pomaria_ele.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:397d10e0125fba7d3cbf512b391a6482697d69edfb8700e089746f8ef41db506
+size 9534331

rir/ours/combined_pomaria_tx_moving_ele.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7906b8219868ace909e7611853525db4ed58f32d0723684bb76b2745628be9c4
+size 9600134

rir/ours/combined_pomaria_tx_rotation_ssl.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aebf1533cbda8ee3aa5287324fd7ac4ab145da348e6321b8581ee1526a16bb38
+size 8442679

rir/ours/combined_rs_ele.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8050eafe06330285e049e8aee2b079658c7f42f2064603a16405d3fd4017fdc
+size 9761008

rir/ours/combined_rs_tx_rotation1_ssl.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a24f8cb18cb242a63982133438215f85dc68fb45a1744c5db0df5dc3c78b672a
+size 7458436

rir/ours/combined_rs_tx_rotation2_ssl.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdd53a6d21c1fdce48732b8120b575deaaf5e30679b8928c139c3a3608f75d1b
+size 6808873

rir/rir.json

@@ -0,0 +1,42 @@
+{
+    "bene": {
+        "ours": "rir/ours/combined_bene_ele.mp4",
+        "baseline": "rir/baseline/combined_bene_vanilla.mp4",
+        "groundtruth": "rir/groundtruth/combined_bene_gt.mp4"
+    },
+    "bene_tx_rotation": {
+        "ours": "rir/ours/combined_bene_tx_rotation_ssl.mp4",
+        "baseline": "rir/baseline/combined_bene_tx_rotation_ele.mp4",
+        "groundtruth": "rir/groundtruth/combined_bene_tx_rotation_gt.mp4"
+    },
+    "pomaria": {
+        "ours": "rir/ours/combined_pomaria_ele.mp4",
+        "baseline": "rir/baseline/combined_pomaria_vanilla.mp4",
+        "groundtruth": "rir/groundtruth/combined_pomaria_gt.mp4"
+    },
+    "pomaria_tx_rotation": {
+        "ours": "rir/ours/combined_pomaria_tx_rotation_ssl.mp4",
+        "baseline": "rir/baseline/combined_pomaria_tx_rotation_ele.mp4",
+        "groundtruth": "rir/groundtruth/combined_pomaria_tx_rotation_gt.mp4"
+    },
+    "pomaria_tx_moving": {
+        "ours": "rir/ours/combined_pomaria_tx_moving_ele.mp4",
+        "baseline": "rir/baseline/combined_pomaria_tx_moving_vanilla.mp4",
+        "groundtruth": "rir/groundtruth/combined_pomaria_tx_moving_gt.mp4"
+    },
+    "rs": {
+        "ours": "rir/ours/combined_rs_ele.mp4",
+        "baseline": "rir/baseline/combined_rs_vanilla.mp4",
+        "groundtruth": "rir/groundtruth/combined_rs_gt.mp4"
+    },
+    "rs_tx_rotation1": {
+        "ours": "rir/ours/combined_rs_tx_rotation1_ssl.mp4",
+        "baseline": "rir/baseline/combined_rs_tx_rotation1_ele.mp4",
+        "groundtruth": "rir/groundtruth/combined_rs_tx_rotation1_gt.mp4"
+    },
+    "rs_tx_rotation2": {
+        "ours": "rir/ours/combined_rs_tx_rotation2_ssl.mp4",
+        "baseline": "rir/baseline/combined_rs_tx_rotation2_ele.mp4",
+        "groundtruth": "rir/groundtruth/combined_rs_tx_rotation2_gt.mp4"
+    }
+}