text/symbols.py

punctuation = ["!", "?", "…", ",", ".", "'", "-"]
pu_symbols = punctuation + ["SP", "UNK"]
pad = "_"

# chinese
zh_symbols = [
    "E", "En", "a", "ai", "an", "ang", "ao", "b", "c", "ch", "d", "e", "ei", "en", "eng", "er", "f", "g", "h", "i", "i0", "ia", "ian", "iang", "iao", "ie",
    "in", "ing", "iong", "ir", "iu", "j", "k", "l", "m", "n", "o", "ong", "ou", "p", "q", "r", "s", "sh", "t", "u", "ua", "uai", "uan", "uang", "ui", "un",
    "uo", "v", "van", "ve", "vn", "w", "x", "y", "z", "zh", "AA", "EE", "OO",
]
zh_symbols = ["ZH_" + i for i in zh_symbols]
num_zh_tones = 6

# shanghainese
sh_symbols = [
    "\u2026", "a", "b", "d", "f", "g", "h", "i", "k", "l", "m", "n", "o", "p", "s", "t", "u", "v", "y", "z", "\u00f8", "\u014b", "\u0235", "\u0251", "\u0254",
    "\u0255", "\u0259", "\u0264", "\u0266", "\u026a", "\u027f", "\u0291", "\u0294", "\u02b0", "\u0303", "\u0329", "\u1d00", "\u1d07"
]
sh_symbols = ["SH_" + i for i in sh_symbols]
num_sh_tones = 9

# English
en_symbols = [
    "aa", "ae", "ah", "ao", "aw", "ay", "b", "ch", "d", "dh", "eh", "er", "ey", "f", "g", "hh", "ih", "iy", "jh", "k", "l", "m", "n", "ng", "ow", "oy", "p",
    "r", "s", "sh", "t", "th", "uh", "uw", "V", "w", "y", "z", "zh",
]
en_symbols = ["EN_" + i for i in en_symbols]
num_en_tones = 4

# combine all symbols
normal_symbols = sorted(set(zh_symbols + sh_symbols + en_symbols))
symbols = [pad] + normal_symbols + pu_symbols
sil_phonemes_ids = [symbols.index(i) for i in pu_symbols]

# combine all tones
num_tones = num_zh_tones + num_sh_tones + num_en_tones

# language maps
language_id_map = {"ZH": 0, "SH": 1, "EN": 2}
num_languages = len(language_id_map.keys())

language_tone_start_map = {
    "ZH": 0,
    "SH": num_zh_tones,
    "EN": num_zh_tones + num_sh_tones,
}

if __name__ == "__main__":
    a = set(zh_symbols)
    b = set(sh_symbols)
    c = set(en_symbols)
    print(sorted(a & b & c))