Spaces:
Runtime error
Runtime error
Echo
commited on
Commit
·
5bf8c6d
1
Parent(s):
053e485
upgrade bittensor & enhance async fetch subnet 6
Browse files- app.py +30 -19
- backup/README.md +0 -12
- backup/app.py +0 -406
- backup/nousgirl.png +0 -0
- backup/requirements.txt +0 -7
- requirements.txt +2 -2
app.py
CHANGED
@@ -15,19 +15,20 @@ from dotenv import load_dotenv
|
|
15 |
from huggingface_hub import HfApi
|
16 |
from apscheduler.schedulers.background import BackgroundScheduler
|
17 |
from tqdm import tqdm
|
|
|
18 |
|
19 |
load_dotenv()
|
20 |
|
21 |
FONT = """<link href="https://fonts.cdnfonts.com/css/jmh-typewriter" rel="stylesheet">"""
|
22 |
-
TITLE = """<h1 align="center" id="space-title" class="typewriter">Subnet 6 Leaderboard</h1>"""
|
23 |
IMAGE = """<a href="https://discord.gg/jqVphNsB4H" target="_blank"><img src="https://i.ibb.co/88wyVQ7/nousgirl.png" alt="nousgirl" style="margin: auto; width: 20%; border: 0;" /></a>"""
|
24 |
HEADER = """<h2 align="center" class="typewriter"><a href="https://github.com/NousResearch/finetuning-subnet" target="_blank">Subnet 6</a> is a <a href="https://bittensor.com/" target="_blank">Bittensor</a> subnet that incentivizes the creation of the best open models by evaluating submissions on a constant stream of newly generated synthetic GPT-4 data. The models with the best <a href="https://github.com/NousResearch/finetuning-subnet/blob/master/docs/validator.md" target="_blank">head-to-head loss</a> on the evaluation data receive a steady emission of TAO.</h3>"""
|
25 |
EVALUATION_DETAILS = """<b>Name</b> is the 🤗 Hugging Face model name (click to go to the model card). <b>Rewards / Day</b> are the expected rewards per day for each model. <b>Perplexity</b> is represents the loss on all of the evaluation data for the model as calculated by the validator (lower is better). <b>UID</b> is the Bittensor user id of the submitter. <b>Block</b> is the Bittensor block that the model was submitted in. More stats on <a href="https://taostats.io/subnets/netuid-6/" target="_blank">taostats</a>."""
|
26 |
-
EVALUATION_HEADER = """<h3 align="center">Shows the latest internal evaluation statistics as calculated by a validator run by
|
27 |
VALIDATOR_WANDB_PROJECT = os.environ["VALIDATOR_WANDB_PROJECT"]
|
28 |
H4_TOKEN = os.environ.get("H4_TOKEN", None)
|
29 |
API = HfApi(token=H4_TOKEN)
|
30 |
-
REPO_ID = "
|
31 |
METAGRAPH_RETRIES = 10
|
32 |
METAGRAPH_DELAY_SECS = 30
|
33 |
METADATA_TTL = 10
|
@@ -43,6 +44,7 @@ class Competition:
|
|
43 |
|
44 |
COMPETITIONS = [Competition(id="m1", name="mistral-7b"), Competition(id="g1", name="gemma-2b")]
|
45 |
DEFAULT_COMPETITION_ID = "m1"
|
|
|
46 |
|
47 |
def run_in_subprocess(func: functools.partial, ttl: int) -> typing.Any:
|
48 |
"""Runs the provided function on a subprocess with 'ttl' seconds to complete.
|
@@ -160,19 +162,18 @@ def get_validator_weights(metagraph: bt.metagraph) -> typing.Dict[int, typing.Tu
|
|
160 |
return ret
|
161 |
|
162 |
def get_subnet_data(subtensor: bt.subtensor, metagraph: bt.metagraph) -> typing.List[ModelData]:
|
163 |
-
|
164 |
-
|
|
|
165 |
hotkey = metagraph.hotkeys[uid]
|
166 |
try:
|
167 |
-
# Wrap calls to the subtensor in a subprocess with a timeout to handle potential hangs.
|
168 |
partial = functools.partial(get_metadata, subtensor, metagraph.netuid, hotkey)
|
169 |
metadata = run_in_subprocess(partial, METADATA_TTL)
|
170 |
-
except
|
171 |
-
|
172 |
-
|
173 |
-
metadata = None
|
174 |
if not metadata:
|
175 |
-
|
176 |
|
177 |
commitment = metadata["info"]["fields"][0]
|
178 |
hex_data = commitment[list(commitment.keys())[0]][2:]
|
@@ -181,14 +182,25 @@ def get_subnet_data(subtensor: bt.subtensor, metagraph: bt.metagraph) -> typing.
|
|
181 |
incentive = metagraph.incentive[uid].nan_to_num().item()
|
182 |
emission = metagraph.emission[uid].nan_to_num().item() * 20 # convert to daily TAO
|
183 |
|
184 |
-
model_data = None
|
185 |
try:
|
186 |
model_data = ModelData.from_compressed_str(uid, hotkey, chain_str, block, incentive, emission)
|
187 |
-
except:
|
188 |
-
|
|
|
189 |
|
190 |
-
|
191 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
192 |
|
193 |
def floatable(x) -> bool:
|
194 |
return (isinstance(x, float) and not math.isnan(x) and not math.isinf(x)) or isinstance(x, int)
|
@@ -345,8 +357,7 @@ with demo:
|
|
345 |
)
|
346 |
|
347 |
with gr.Accordion("Evaluation Stats"):
|
348 |
-
gr.HTML(EVALUATION_HEADER)
|
349 |
-
|
350 |
with gr.Tabs():
|
351 |
for entry in leaderboard_df:
|
352 |
if entry.competition == competition.id:
|
@@ -389,7 +400,7 @@ def restart_space():
|
|
389 |
API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)
|
390 |
|
391 |
scheduler = BackgroundScheduler()
|
392 |
-
scheduler.add_job(restart_space, "interval", seconds=60
|
393 |
scheduler.start()
|
394 |
|
395 |
demo.launch()
|
|
|
15 |
from huggingface_hub import HfApi
|
16 |
from apscheduler.schedulers.background import BackgroundScheduler
|
17 |
from tqdm import tqdm
|
18 |
+
import concurrent.futures
|
19 |
|
20 |
load_dotenv()
|
21 |
|
22 |
FONT = """<link href="https://fonts.cdnfonts.com/css/jmh-typewriter" rel="stylesheet">"""
|
23 |
+
TITLE = """<h1 align="center" id="space-title" class="typewriter">Subnet 6 Leaderboard (with Cortex Foundation validator/subtensor)</h1>"""
|
24 |
IMAGE = """<a href="https://discord.gg/jqVphNsB4H" target="_blank"><img src="https://i.ibb.co/88wyVQ7/nousgirl.png" alt="nousgirl" style="margin: auto; width: 20%; border: 0;" /></a>"""
|
25 |
HEADER = """<h2 align="center" class="typewriter"><a href="https://github.com/NousResearch/finetuning-subnet" target="_blank">Subnet 6</a> is a <a href="https://bittensor.com/" target="_blank">Bittensor</a> subnet that incentivizes the creation of the best open models by evaluating submissions on a constant stream of newly generated synthetic GPT-4 data. The models with the best <a href="https://github.com/NousResearch/finetuning-subnet/blob/master/docs/validator.md" target="_blank">head-to-head loss</a> on the evaluation data receive a steady emission of TAO.</h3>"""
|
26 |
EVALUATION_DETAILS = """<b>Name</b> is the 🤗 Hugging Face model name (click to go to the model card). <b>Rewards / Day</b> are the expected rewards per day for each model. <b>Perplexity</b> is represents the loss on all of the evaluation data for the model as calculated by the validator (lower is better). <b>UID</b> is the Bittensor user id of the submitter. <b>Block</b> is the Bittensor block that the model was submitted in. More stats on <a href="https://taostats.io/subnets/netuid-6/" target="_blank">taostats</a>."""
|
27 |
+
EVALUATION_HEADER = """<h3 align="center">Shows the latest internal evaluation statistics as calculated by a validator run by Cortex Foundation ({date}) </h3>"""
|
28 |
VALIDATOR_WANDB_PROJECT = os.environ["VALIDATOR_WANDB_PROJECT"]
|
29 |
H4_TOKEN = os.environ.get("H4_TOKEN", None)
|
30 |
API = HfApi(token=H4_TOKEN)
|
31 |
+
REPO_ID = "0x9/finetuning_subnet_leaderboard"
|
32 |
METAGRAPH_RETRIES = 10
|
33 |
METAGRAPH_DELAY_SECS = 30
|
34 |
METADATA_TTL = 10
|
|
|
44 |
|
45 |
COMPETITIONS = [Competition(id="m1", name="mistral-7b"), Competition(id="g1", name="gemma-2b")]
|
46 |
DEFAULT_COMPETITION_ID = "m1"
|
47 |
+
last_refresh = None
|
48 |
|
49 |
def run_in_subprocess(func: functools.partial, ttl: int) -> typing.Any:
|
50 |
"""Runs the provided function on a subprocess with 'ttl' seconds to complete.
|
|
|
162 |
return ret
|
163 |
|
164 |
def get_subnet_data(subtensor: bt.subtensor, metagraph: bt.metagraph) -> typing.List[ModelData]:
|
165 |
+
global last_refresh
|
166 |
+
# Function to be executed in a thread
|
167 |
+
def fetch_data(uid):
|
168 |
hotkey = metagraph.hotkeys[uid]
|
169 |
try:
|
|
|
170 |
partial = functools.partial(get_metadata, subtensor, metagraph.netuid, hotkey)
|
171 |
metadata = run_in_subprocess(partial, METADATA_TTL)
|
172 |
+
except Exception as e:
|
173 |
+
return None
|
174 |
+
|
|
|
175 |
if not metadata:
|
176 |
+
return None
|
177 |
|
178 |
commitment = metadata["info"]["fields"][0]
|
179 |
hex_data = commitment[list(commitment.keys())[0]][2:]
|
|
|
182 |
incentive = metagraph.incentive[uid].nan_to_num().item()
|
183 |
emission = metagraph.emission[uid].nan_to_num().item() * 20 # convert to daily TAO
|
184 |
|
|
|
185 |
try:
|
186 |
model_data = ModelData.from_compressed_str(uid, hotkey, chain_str, block, incentive, emission)
|
187 |
+
except Exception as e:
|
188 |
+
return None
|
189 |
+
return model_data
|
190 |
|
191 |
+
# Use ThreadPoolExecutor to fetch data in parallel
|
192 |
+
results = []
|
193 |
+
with concurrent.futures.ThreadPoolExecutor() as executor:
|
194 |
+
# Prepare the list of futures
|
195 |
+
futures = [executor.submit(fetch_data, uid) for uid in metagraph.uids.tolist()]
|
196 |
+
for future in tqdm(concurrent.futures.as_completed(futures), desc="Metadata for hotkeys", total=len(futures)):
|
197 |
+
result = future.result()
|
198 |
+
if result:
|
199 |
+
results.append(result)
|
200 |
+
|
201 |
+
|
202 |
+
last_refresh = datetime.datetime.now()
|
203 |
+
return results
|
204 |
|
205 |
def floatable(x) -> bool:
|
206 |
return (isinstance(x, float) and not math.isnan(x) and not math.isinf(x)) or isinstance(x, int)
|
|
|
357 |
)
|
358 |
|
359 |
with gr.Accordion("Evaluation Stats"):
|
360 |
+
gr.HTML(EVALUATION_HEADER.replace("{date}", last_refresh.strftime("refreshed at %H:%M on %Y-%m-%d")))
|
|
|
361 |
with gr.Tabs():
|
362 |
for entry in leaderboard_df:
|
363 |
if entry.competition == competition.id:
|
|
|
400 |
API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)
|
401 |
|
402 |
scheduler = BackgroundScheduler()
|
403 |
+
scheduler.add_job(restart_space, "interval", seconds=60*5) # restart every 15 minutes
|
404 |
scheduler.start()
|
405 |
|
406 |
demo.launch()
|
backup/README.md
DELETED
@@ -1,12 +0,0 @@
|
|
1 |
-
---
|
2 |
-
title: Finetuning Subnet Leaderboard
|
3 |
-
emoji: ⚒️
|
4 |
-
colorFrom: indigo
|
5 |
-
colorTo: blue
|
6 |
-
sdk: gradio
|
7 |
-
sdk_version: 3.41.0
|
8 |
-
app_file: app.py
|
9 |
-
pinned: false
|
10 |
-
---
|
11 |
-
|
12 |
-
Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
backup/app.py
DELETED
@@ -1,406 +0,0 @@
|
|
1 |
-
import gradio as gr
|
2 |
-
import bittensor as bt
|
3 |
-
import typing
|
4 |
-
from bittensor.extrinsics.serving import get_metadata
|
5 |
-
from dataclasses import dataclass
|
6 |
-
import requests
|
7 |
-
import wandb
|
8 |
-
import math
|
9 |
-
import os
|
10 |
-
import datetime
|
11 |
-
import time
|
12 |
-
import functools
|
13 |
-
import multiprocessing
|
14 |
-
from dotenv import load_dotenv
|
15 |
-
from huggingface_hub import HfApi
|
16 |
-
from apscheduler.schedulers.background import BackgroundScheduler
|
17 |
-
from tqdm import tqdm
|
18 |
-
import concurrent.futures
|
19 |
-
|
20 |
-
load_dotenv()
|
21 |
-
|
22 |
-
FONT = """<link href="https://fonts.cdnfonts.com/css/jmh-typewriter" rel="stylesheet">"""
|
23 |
-
TITLE = """<h1 align="center" id="space-title" class="typewriter">Subnet 6 Leaderboard (with Cortex Foundation validator/subtensor)</h1>"""
|
24 |
-
IMAGE = """<a href="https://discord.gg/jqVphNsB4H" target="_blank"><img src="https://i.ibb.co/88wyVQ7/nousgirl.png" alt="nousgirl" style="margin: auto; width: 20%; border: 0;" /></a>"""
|
25 |
-
HEADER = """<h2 align="center" class="typewriter"><a href="https://github.com/NousResearch/finetuning-subnet" target="_blank">Subnet 6</a> is a <a href="https://bittensor.com/" target="_blank">Bittensor</a> subnet that incentivizes the creation of the best open models by evaluating submissions on a constant stream of newly generated synthetic GPT-4 data. The models with the best <a href="https://github.com/NousResearch/finetuning-subnet/blob/master/docs/validator.md" target="_blank">head-to-head loss</a> on the evaluation data receive a steady emission of TAO.</h3>"""
|
26 |
-
EVALUATION_DETAILS = """<b>Name</b> is the 🤗 Hugging Face model name (click to go to the model card). <b>Rewards / Day</b> are the expected rewards per day for each model. <b>Perplexity</b> is represents the loss on all of the evaluation data for the model as calculated by the validator (lower is better). <b>UID</b> is the Bittensor user id of the submitter. <b>Block</b> is the Bittensor block that the model was submitted in. More stats on <a href="https://taostats.io/subnets/netuid-6/" target="_blank">taostats</a>."""
|
27 |
-
EVALUATION_HEADER = """<h3 align="center">Shows the latest internal evaluation statistics as calculated by a validator run by Cortex Foundation ({date}) </h3>"""
|
28 |
-
VALIDATOR_WANDB_PROJECT = os.environ["VALIDATOR_WANDB_PROJECT"]
|
29 |
-
H4_TOKEN = os.environ.get("H4_TOKEN", None)
|
30 |
-
API = HfApi(token=H4_TOKEN)
|
31 |
-
REPO_ID = "0x9/finetuning_subnet_leaderboard"
|
32 |
-
METAGRAPH_RETRIES = 10
|
33 |
-
METAGRAPH_DELAY_SECS = 30
|
34 |
-
METADATA_TTL = 10
|
35 |
-
NETUID = 6
|
36 |
-
SUBNET_START_BLOCK = 2225782
|
37 |
-
SECONDS_PER_BLOCK = 12
|
38 |
-
SUBTENSOR = os.environ.get("SUBTENSOR", "finney")
|
39 |
-
|
40 |
-
@dataclass
|
41 |
-
class Competition:
|
42 |
-
id: str
|
43 |
-
name: str
|
44 |
-
|
45 |
-
COMPETITIONS = [Competition(id="m1", name="mistral-7b"), Competition(id="g1", name="gemma-2b")]
|
46 |
-
DEFAULT_COMPETITION_ID = "m1"
|
47 |
-
last_refresh = None
|
48 |
-
|
49 |
-
def run_in_subprocess(func: functools.partial, ttl: int) -> typing.Any:
|
50 |
-
"""Runs the provided function on a subprocess with 'ttl' seconds to complete.
|
51 |
-
|
52 |
-
Args:
|
53 |
-
func (functools.partial): Function to be run.
|
54 |
-
ttl (int): How long to try for in seconds.
|
55 |
-
|
56 |
-
Returns:
|
57 |
-
Any: The value returned by 'func'
|
58 |
-
"""
|
59 |
-
|
60 |
-
def wrapped_func(func: functools.partial, queue: multiprocessing.Queue):
|
61 |
-
try:
|
62 |
-
result = func()
|
63 |
-
queue.put(result)
|
64 |
-
except (Exception, BaseException) as e:
|
65 |
-
# Catch exceptions here to add them to the queue.
|
66 |
-
queue.put(e)
|
67 |
-
|
68 |
-
# Use "fork" (the default on all POSIX except macOS), because pickling doesn't seem
|
69 |
-
# to work on "spawn".
|
70 |
-
ctx = multiprocessing.get_context("fork")
|
71 |
-
queue = ctx.Queue()
|
72 |
-
process = ctx.Process(target=wrapped_func, args=[func, queue])
|
73 |
-
|
74 |
-
process.start()
|
75 |
-
|
76 |
-
process.join(timeout=ttl)
|
77 |
-
|
78 |
-
if process.is_alive():
|
79 |
-
process.terminate()
|
80 |
-
process.join()
|
81 |
-
raise TimeoutError(f"Failed to {func.func.__name__} after {ttl} seconds")
|
82 |
-
|
83 |
-
# Raises an error if the queue is empty. This is fine. It means our subprocess timed out.
|
84 |
-
result = queue.get(block=False)
|
85 |
-
|
86 |
-
# If we put an exception on the queue then raise instead of returning.
|
87 |
-
if isinstance(result, Exception):
|
88 |
-
raise result
|
89 |
-
if isinstance(result, BaseException):
|
90 |
-
raise Exception(f"BaseException raised in subprocess: {str(result)}")
|
91 |
-
|
92 |
-
return result
|
93 |
-
|
94 |
-
|
95 |
-
def get_subtensor_and_metagraph() -> typing.Tuple[bt.subtensor, bt.metagraph]:
|
96 |
-
for i in range(0, METAGRAPH_RETRIES):
|
97 |
-
try:
|
98 |
-
print("Connecting to subtensor...")
|
99 |
-
subtensor: bt.subtensor = bt.subtensor(SUBTENSOR)
|
100 |
-
print("Pulling metagraph...")
|
101 |
-
metagraph: bt.metagraph = subtensor.metagraph(NETUID, lite=False)
|
102 |
-
return subtensor, metagraph
|
103 |
-
except:
|
104 |
-
if i == METAGRAPH_RETRIES - 1:
|
105 |
-
raise
|
106 |
-
print(f"Error connecting to subtensor or pulling metagraph, retry {i + 1} of {METAGRAPH_RETRIES} in {METAGRAPH_DELAY_SECS} seconds...")
|
107 |
-
time.sleep(METAGRAPH_DELAY_SECS)
|
108 |
-
raise RuntimeError()
|
109 |
-
|
110 |
-
@dataclass
|
111 |
-
class ModelData:
|
112 |
-
uid: int
|
113 |
-
hotkey: str
|
114 |
-
namespace: str
|
115 |
-
name: str
|
116 |
-
commit: str
|
117 |
-
hash: str
|
118 |
-
block: int
|
119 |
-
incentive: float
|
120 |
-
emission: float
|
121 |
-
competition: str
|
122 |
-
|
123 |
-
@classmethod
|
124 |
-
def from_compressed_str(cls, uid: int, hotkey: str, cs: str, block: int, incentive: float, emission: float):
|
125 |
-
"""Returns an instance of this class from a compressed string representation"""
|
126 |
-
tokens = cs.split(":")
|
127 |
-
return ModelData(
|
128 |
-
uid=uid,
|
129 |
-
hotkey=hotkey,
|
130 |
-
namespace=tokens[0],
|
131 |
-
name=tokens[1],
|
132 |
-
commit=tokens[2] if tokens[2] != "None" else "",
|
133 |
-
hash=tokens[3] if tokens[3] != "None" else "",
|
134 |
-
competition=tokens[4] if len(tokens) > 4 and tokens[4] != "None" else DEFAULT_COMPETITION_ID,
|
135 |
-
block=block,
|
136 |
-
incentive=incentive,
|
137 |
-
emission=emission
|
138 |
-
)
|
139 |
-
|
140 |
-
def get_tao_price() -> float:
|
141 |
-
for i in range(0, METAGRAPH_RETRIES):
|
142 |
-
try:
|
143 |
-
return float(requests.get("https://api.kucoin.com/api/v1/market/stats?symbol=TAO-USDT").json()["data"]["last"])
|
144 |
-
except:
|
145 |
-
if i == METAGRAPH_RETRIES - 1:
|
146 |
-
raise
|
147 |
-
time.sleep(METAGRAPH_DELAY_SECS)
|
148 |
-
raise RuntimeError()
|
149 |
-
|
150 |
-
def get_validator_weights(metagraph: bt.metagraph) -> typing.Dict[int, typing.Tuple[float, int, typing.Dict[int, float]]]:
|
151 |
-
ret = {}
|
152 |
-
for uid in metagraph.uids.tolist():
|
153 |
-
vtrust = metagraph.validator_trust[uid].item()
|
154 |
-
if vtrust > 0:
|
155 |
-
ret[uid] = (vtrust, metagraph.S[uid].item(), {})
|
156 |
-
for ouid in metagraph.uids.tolist():
|
157 |
-
if ouid == uid:
|
158 |
-
continue
|
159 |
-
weight = round(metagraph.weights[uid][ouid].item(), 4)
|
160 |
-
if weight > 0:
|
161 |
-
ret[uid][-1][ouid] = weight
|
162 |
-
return ret
|
163 |
-
|
164 |
-
def get_subnet_data(subtensor: bt.subtensor, metagraph: bt.metagraph) -> typing.List[ModelData]:
|
165 |
-
global last_refresh
|
166 |
-
# Function to be executed in a thread
|
167 |
-
def fetch_data(uid):
|
168 |
-
hotkey = metagraph.hotkeys[uid]
|
169 |
-
try:
|
170 |
-
partial = functools.partial(get_metadata, subtensor, metagraph.netuid, hotkey)
|
171 |
-
metadata = run_in_subprocess(partial, METADATA_TTL)
|
172 |
-
except Exception as e:
|
173 |
-
return None
|
174 |
-
|
175 |
-
if not metadata:
|
176 |
-
return None
|
177 |
-
|
178 |
-
commitment = metadata["info"]["fields"][0]
|
179 |
-
hex_data = commitment[list(commitment.keys())[0]][2:]
|
180 |
-
chain_str = bytes.fromhex(hex_data).decode()
|
181 |
-
block = metadata["block"]
|
182 |
-
incentive = metagraph.incentive[uid].nan_to_num().item()
|
183 |
-
emission = metagraph.emission[uid].nan_to_num().item() * 20 # convert to daily TAO
|
184 |
-
|
185 |
-
try:
|
186 |
-
model_data = ModelData.from_compressed_str(uid, hotkey, chain_str, block, incentive, emission)
|
187 |
-
except Exception as e:
|
188 |
-
return None
|
189 |
-
return model_data
|
190 |
-
|
191 |
-
# Use ThreadPoolExecutor to fetch data in parallel
|
192 |
-
results = []
|
193 |
-
with concurrent.futures.ThreadPoolExecutor() as executor:
|
194 |
-
# Prepare the list of futures
|
195 |
-
futures = [executor.submit(fetch_data, uid) for uid in metagraph.uids.tolist()]
|
196 |
-
for future in tqdm(concurrent.futures.as_completed(futures), desc="Metadata for hotkeys", total=len(futures)):
|
197 |
-
result = future.result()
|
198 |
-
if result:
|
199 |
-
results.append(result)
|
200 |
-
|
201 |
-
|
202 |
-
last_refresh = datetime.datetime.now()
|
203 |
-
return results
|
204 |
-
|
205 |
-
def floatable(x) -> bool:
|
206 |
-
return (isinstance(x, float) and not math.isnan(x) and not math.isinf(x)) or isinstance(x, int)
|
207 |
-
|
208 |
-
def get_float_score(key: str, history, competition_id: str) -> typing.Tuple[typing.Optional[float], bool]:
|
209 |
-
if key in history and "competition_id" in history:
|
210 |
-
data = list(history[key])
|
211 |
-
if len(data) > 0:
|
212 |
-
competitions = list(history["competition_id"])
|
213 |
-
while True:
|
214 |
-
if competitions.pop() != competition_id:
|
215 |
-
data.pop()
|
216 |
-
continue
|
217 |
-
if floatable(data[-1]):
|
218 |
-
return float(data[-1]), True
|
219 |
-
else:
|
220 |
-
data = [float(x) for x in data if floatable(x)]
|
221 |
-
if len(data) > 0:
|
222 |
-
return float(data[-1]), False
|
223 |
-
break
|
224 |
-
return None, False
|
225 |
-
|
226 |
-
def get_sample(uid, history, competition_id: str) -> typing.Optional[typing.Tuple[str, str, str]]:
|
227 |
-
prompt_key = f"sample_prompt_data.{uid}"
|
228 |
-
response_key = f"sample_response_data.{uid}"
|
229 |
-
truth_key = f"sample_truth_data.{uid}"
|
230 |
-
if prompt_key in history and response_key in history and truth_key in history and "competition_id" in history:
|
231 |
-
competitions = list(history["competition_id"])
|
232 |
-
prompts = list(history[prompt_key])
|
233 |
-
responses = list(history[response_key])
|
234 |
-
truths = list(history[truth_key])
|
235 |
-
while True:
|
236 |
-
prompt = prompts.pop()
|
237 |
-
response = responses.pop()
|
238 |
-
truth = truths.pop()
|
239 |
-
if competitions.pop() != competition_id:
|
240 |
-
continue
|
241 |
-
if isinstance(prompt, str) and isinstance(response, str) and isinstance(truth, str):
|
242 |
-
return prompt, response, truth
|
243 |
-
break
|
244 |
-
return None
|
245 |
-
|
246 |
-
def get_scores(uids: typing.List[int], competition_id: str) -> typing.Dict[int, typing.Dict[str, typing.Optional[float | str]]]:
|
247 |
-
api = wandb.Api()
|
248 |
-
runs = list(api.runs(VALIDATOR_WANDB_PROJECT))
|
249 |
-
|
250 |
-
result = {}
|
251 |
-
for run in runs:
|
252 |
-
history = run.history()
|
253 |
-
for uid in uids:
|
254 |
-
if uid in result.keys():
|
255 |
-
continue
|
256 |
-
perplexity, perplexity_fresh = get_float_score(f"perplexity_data.{uid}", history, competition_id)
|
257 |
-
win_rate, win_rate_fresh = get_float_score(f"win_rate_data.{uid}", history, competition_id)
|
258 |
-
win_total, win_total_fresh = get_float_score(f"win_total_data.{uid}", history, competition_id)
|
259 |
-
weight, weight_fresh = get_float_score(f"weight_data.{uid}", history, competition_id)
|
260 |
-
sample = get_sample(uid, history, competition_id)
|
261 |
-
result[uid] = {
|
262 |
-
"perplexity": perplexity,
|
263 |
-
"win_rate": win_rate,
|
264 |
-
"win_total": win_total,
|
265 |
-
"weight": weight,
|
266 |
-
"sample": sample,
|
267 |
-
"fresh": perplexity_fresh and win_rate_fresh and win_total_fresh
|
268 |
-
}
|
269 |
-
if len(result.keys()) == len(uids):
|
270 |
-
break
|
271 |
-
return result
|
272 |
-
|
273 |
-
def format_score(uid, scores, key) -> typing.Optional[float]:
|
274 |
-
if uid in scores:
|
275 |
-
if key in scores[uid]:
|
276 |
-
point = scores[uid][key]
|
277 |
-
if floatable(point):
|
278 |
-
return round(scores[uid][key], 4)
|
279 |
-
return None
|
280 |
-
|
281 |
-
def next_tempo(start_block, tempo, block):
|
282 |
-
start_num = start_block + tempo
|
283 |
-
intervals = (block - start_num) // tempo
|
284 |
-
nearest_num = start_num + ((intervals + 1) * tempo)
|
285 |
-
return nearest_num
|
286 |
-
|
287 |
-
subtensor, metagraph = get_subtensor_and_metagraph()
|
288 |
-
|
289 |
-
tao_price = get_tao_price()
|
290 |
-
|
291 |
-
leaderboard_df = get_subnet_data(subtensor, metagraph)
|
292 |
-
leaderboard_df.sort(key=lambda x: x.incentive, reverse=True)
|
293 |
-
|
294 |
-
competition_scores = {
|
295 |
-
y.id: get_scores([x.uid for x in leaderboard_df if x.competition == y.id], y.id)
|
296 |
-
for y in COMPETITIONS
|
297 |
-
}
|
298 |
-
|
299 |
-
current_block = metagraph.block.item()
|
300 |
-
next_update = next_tempo(
|
301 |
-
SUBNET_START_BLOCK,
|
302 |
-
subtensor.get_subnet_hyperparameters(NETUID).tempo,
|
303 |
-
current_block
|
304 |
-
)
|
305 |
-
blocks_to_go = next_update - current_block
|
306 |
-
current_time = datetime.datetime.now()
|
307 |
-
next_update_time = current_time + datetime.timedelta(seconds=blocks_to_go * SECONDS_PER_BLOCK)
|
308 |
-
|
309 |
-
validator_df = get_validator_weights(metagraph)
|
310 |
-
weight_keys = set()
|
311 |
-
for uid, stats in validator_df.items():
|
312 |
-
weight_keys.update(stats[-1].keys())
|
313 |
-
|
314 |
-
def get_next_update():
|
315 |
-
now = datetime.datetime.now()
|
316 |
-
delta = next_update_time - now
|
317 |
-
return f"""<div align="center" style="font-size: larger;">Next reward update: <b>{blocks_to_go}</b> blocks (~{int(delta.total_seconds() // 60)} minutes)</div>"""
|
318 |
-
|
319 |
-
def leaderboard_data(show_stale: bool, scores: typing.Dict[int, typing.Dict[str, typing.Optional[float | str]]], competition_id: str):
|
320 |
-
value = [
|
321 |
-
[
|
322 |
-
f'[{c.namespace}/{c.name} ({c.commit[0:8]}, UID={c.uid})](https://huggingface.co/{c.namespace}/{c.name}/commit/{c.commit})',
|
323 |
-
format_score(c.uid, scores, "win_rate"),
|
324 |
-
format_score(c.uid, scores, "perplexity"),
|
325 |
-
format_score(c.uid, scores, "weight"),
|
326 |
-
c.uid,
|
327 |
-
c.block
|
328 |
-
] for c in leaderboard_df if c.competition == competition_id and (scores[c.uid]["fresh"] or show_stale)
|
329 |
-
]
|
330 |
-
return value
|
331 |
-
|
332 |
-
demo = gr.Blocks(css=".typewriter {font-family: 'JMH Typewriter', sans-serif;}")
|
333 |
-
with demo:
|
334 |
-
gr.HTML(FONT)
|
335 |
-
gr.HTML(TITLE)
|
336 |
-
gr.HTML(IMAGE)
|
337 |
-
gr.HTML(HEADER)
|
338 |
-
|
339 |
-
gr.HTML(value=get_next_update())
|
340 |
-
|
341 |
-
with gr.Tabs():
|
342 |
-
for competition in COMPETITIONS:
|
343 |
-
with gr.Tab(competition.name):
|
344 |
-
scores = competition_scores[competition.id]
|
345 |
-
print(scores)
|
346 |
-
|
347 |
-
class_denominator = sum(leaderboard_df[i].incentive for i in range(0, 10) if leaderboard_df[i].incentive and leaderboard_df[i].competition == competition.id)
|
348 |
-
|
349 |
-
class_values = {
|
350 |
-
f"{leaderboard_df[i].namespace}/{leaderboard_df[i].name} ({leaderboard_df[i].commit[0:8]}, UID={leaderboard_df[i].uid}) · ${round(leaderboard_df[i].emission * tao_price, 2):,} (τ{round(leaderboard_df[i].emission, 2):,})": \
|
351 |
-
leaderboard_df[i].incentive / class_denominator for i in range(0, 10) if leaderboard_df[i].incentive and leaderboard_df[i].competition == competition.id
|
352 |
-
}
|
353 |
-
|
354 |
-
gr.Label(
|
355 |
-
value=class_values,
|
356 |
-
num_top_classes=10,
|
357 |
-
)
|
358 |
-
|
359 |
-
with gr.Accordion("Evaluation Stats"):
|
360 |
-
gr.HTML(EVALUATION_HEADER.replace("{date}", last_refresh.strftime("refreshed at %H:%M on %Y-%m-%d")))
|
361 |
-
with gr.Tabs():
|
362 |
-
for entry in leaderboard_df:
|
363 |
-
if entry.competition == competition.id:
|
364 |
-
sample = scores[entry.uid]["sample"]
|
365 |
-
if sample is not None:
|
366 |
-
name = f"{entry.namespace}/{entry.name} ({entry.commit[0:8]}, UID={entry.uid})"
|
367 |
-
with gr.Tab(name):
|
368 |
-
gr.Chatbot([(sample[0], sample[1])])
|
369 |
-
# gr.Chatbot([(sample[0], f"*{name}*: {sample[1]}"), (None, f"*GPT-4*: {sample[2]}")])
|
370 |
-
|
371 |
-
show_stale = gr.Checkbox(label="Show Stale", interactive=True)
|
372 |
-
leaderboard_table = gr.components.Dataframe(
|
373 |
-
value=leaderboard_data(show_stale.value, scores, competition.id),
|
374 |
-
headers=["Name", "Win Rate", "Perplexity", "Weight", "UID", "Block"],
|
375 |
-
datatype=["markdown", "number", "number", "number", "number", "number"],
|
376 |
-
elem_id="leaderboard-table",
|
377 |
-
interactive=False,
|
378 |
-
visible=True,
|
379 |
-
)
|
380 |
-
gr.HTML(EVALUATION_DETAILS)
|
381 |
-
show_stale.change(lambda x: leaderboard_data(x, scores, competition.id), [show_stale], leaderboard_table)
|
382 |
-
|
383 |
-
with gr.Accordion("Validator Stats"):
|
384 |
-
validator_table = gr.components.Dataframe(
|
385 |
-
value=[
|
386 |
-
[uid, int(validator_df[uid][1]), round(validator_df[uid][0], 4)] + [validator_df[uid][-1].get(c.uid) for c in leaderboard_df if c.incentive]
|
387 |
-
for uid, _ in sorted(
|
388 |
-
zip(validator_df.keys(), [validator_df[x][1] for x in validator_df.keys()]),
|
389 |
-
key=lambda x: x[1],
|
390 |
-
reverse=True
|
391 |
-
)
|
392 |
-
],
|
393 |
-
headers=["UID", "Stake (τ)", "V-Trust"] + [f"{c.namespace}/{c.name} ({c.commit[0:8]}, UID={c.uid})" for c in leaderboard_df if c.incentive],
|
394 |
-
datatype=["number", "number", "number"] + ["number" for c in leaderboard_df if c.incentive],
|
395 |
-
interactive=False,
|
396 |
-
visible=True,
|
397 |
-
)
|
398 |
-
|
399 |
-
def restart_space():
|
400 |
-
API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)
|
401 |
-
|
402 |
-
scheduler = BackgroundScheduler()
|
403 |
-
scheduler.add_job(restart_space, "interval", seconds=60*5) # restart every 15 minutes
|
404 |
-
scheduler.start()
|
405 |
-
|
406 |
-
demo.launch()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
backup/nousgirl.png
DELETED
Binary file (154 kB)
|
|
backup/requirements.txt
DELETED
@@ -1,7 +0,0 @@
|
|
1 |
-
bittensor==6.9.3
|
2 |
-
requests==2.31.0
|
3 |
-
wandb==0.16.2
|
4 |
-
python-dotenv==1.0.1
|
5 |
-
APScheduler==3.10.1
|
6 |
-
huggingface-hub>=0.18.0
|
7 |
-
tqdm==4.66.2
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
requirements.txt
CHANGED
@@ -1,7 +1,7 @@
|
|
1 |
-
bittensor==6.
|
2 |
requests==2.31.0
|
3 |
wandb==0.16.2
|
4 |
python-dotenv==1.0.1
|
5 |
APScheduler==3.10.1
|
6 |
huggingface-hub>=0.18.0
|
7 |
-
tqdm==4.66.2
|
|
|
1 |
+
bittensor==6.9.3
|
2 |
requests==2.31.0
|
3 |
wandb==0.16.2
|
4 |
python-dotenv==1.0.1
|
5 |
APScheduler==3.10.1
|
6 |
huggingface-hub>=0.18.0
|
7 |
+
tqdm==4.66.2
|