Phonemes and Frequencies, or, Why is TLC's No Scrubs So Catchy?

import urllib.request

CMUDICT_LOCATION = r"http://svn.code.sf.net/p/cmusphinx/code/trunk/cmudict/cmudict-0.7b"

urllib.request.urlretrieve(CMUDICT_LOCATION, r"./cmudict")

cat cmudict | tail -10

cat cmudict | head -10

import re

word_to_pronunciations = {}

# Iterate across every line in the file
with open("cmudict", "r", encoding="latin-1") as fin:
    for line in fin:

        # Split the word off the front of the line
        word, *phonemes = line.split()

        # Skip empty lines
        if len( word ) <= 0:
            continue

        # If this word is punctuation, skip to the next one
        # This skips comment lines too
        first_letter_value = ord(word[0])
        if first_letter_value < ord('A') or first_letter_value > ord('Z'):
            continue

        # Remove the (1), (2) etc from the end of words with mutliple pronunciations
        word = re.sub( r"\(\d+\)$", "", word )

        # Get the current pronunciation list for this word, creating it
        # if needed, and add the new pronunciation.
        try:
            pronunciations = word_to_pronunciations[word]
        except KeyError:
            pronunciations = []
            word_to_pronunciations[ word ] = pronunciations
        pronunciations.append(tuple(phonemes))

word_to_pronunciations[ "TOMATO" ]

VERSE_1 = """
No, I don't want your number
No, I don't wanna give you mine and
No, I don't wanna meet you nowhere
No, I don't want none of your time and

No, I don't want no scrub
A scrub is a guy that can't get no love from me
Hangin' out the passenger side of his best friend's ride
Tryin' to holla at me
"""

LETTERS_AND_APOSTROPHE_RE = r"[^a-zA-Z']"

def text_to_phonemes( text ):
    ret = []
    for line in text.split("\n"):
        for token in re.split("(" + LETTERS_AND_APOSTROPHE_RE + ")", line):
            if token.upper() in word_to_pronunciations:
                # We have a pronunciation for this token -- add it!
                ret.append( "|".join( word_to_pronunciations[ token.upper() ][0] ) )
            else:
                ret.append( token )
        ret.append( "\n" )
    return "".join( ret )

print( text_to_phonemes( VERSE_1 ) )

word_to_pronunciations[ "HOLLA" ] = [ [ "HH", "AA1", "L", "AH0" ] ]

print( text_to_phonemes( VERSE_1 ) )

EXPORT_BOX_STYLE = """
border: 1px solid #ccc;
box-shadow: 3px 3px 3px #eee;
padding: 8pt;
font-family: monospace;
overflow: auto;
margin: 1.2em;"""

WORD_STYLE = """
padding: 1px;
border: 1px solid black;
display: inline-block;"""

VOWEL_PHONEMES = [ "AA", "AE", "AH", "AO", "EH", "IH", "IY", "UH", "UW",
                   "AW", "AY", "ER", "EY", "OW", "OY" ]

def background_color( text, colorcode ):
    return r"""<span style="background-color:#{};">{}</span>""".format( colorcode, text )

def colorize_phonemes( *phonemes ):
    template = r"""<span style="{}">{}</span>"""
    ret = []
    for p in phonemes:
        if p[:-1] in VOWEL_PHONEMES:
            ret.append( background_color( p, "fdd" ) )
        else:
            ret.append( background_color( p, "dfd" ) )
    return template.format( WORD_STYLE, "|".join( ret ) )

def text_to_phonemes( text ):
    template = r"""<div style="{}">{}</div>"""
    ret = []
    for line in text.split("\n"):
        for token in re.split("(" + LETTERS_AND_APOSTROPHE_RE + ")", line):
            if token.upper() in word_to_pronunciations:
                # We have a pronunciation for this token -- add it!
                first_pronunciation = word_to_pronunciations[ token.upper() ][0]
                colored_phonemes = colorize_phonemes( *first_pronunciation )
                ret.append( colored_phonemes )
            else:
                ret.append( token )
        ret.append( "</br>\n" )
    return template.format( EXPORT_BOX_STYLE, "".join( ret ) )

print( text_to_phonemes( VERSE_1 ) )

EXPORT_BOX_STYLE = """
border: 1px solid #ccc;
box-shadow: 3px 3px 3px #eee;
padding: 8pt;
font-family: monospace;
font-size: 13px;
overflow: auto;
margin: 1.2em;"""

WORD_STYLE = """
padding: 1px;
border: 1px solid black;
display: inline-block;"""

VOWEL_PHONEMES = [ "AA", "AE", "AH", "AO", "EH", "IH", "IY", "UH", "UW",
                   "AW", "AY", "ER", "EY", "OW", "OY" ]

def background_color( text, colorcode ):
    return r"""<span style="background-color:#{};">{}</span>""".format( colorcode, text )

def word_to_html( word ):
    template = r"""<span title="{}" style="{}">{}</span>{}"""
    ret = []
    phonemes = word_to_pronunciations[ word.upper() ][0]
    for p in phonemes:
        if p[:-1] in VOWEL_PHONEMES:
            ret.append( background_color( p, "fdd" ) )
        else:
            ret.append( background_color( p, "dfd" ) )
    syllable_count = len( [ p for p in phonemes if p[:-1] in VOWEL_PHONEMES ] )
    return template.format( word, WORD_STYLE, "|".join( ret ), syllable_count )

def text_to_phonemes( text ):
    template = r"""<div style="{}">{}</div>"""
    ret = []
    for line in text.split("\n"):
        for token in re.split("(" + LETTERS_AND_APOSTROPHE_RE + ")", line):
            if token.upper() in word_to_pronunciations:
                # We have a pronunciation for this token -- add it!
                colored_phonemes = word_to_html( token )
                ret.append( colored_phonemes )
            else:
                ret.append( token )
        ret.append( "</br>\n" )
    return template.format( EXPORT_BOX_STYLE, "".join( ret ) )

print( text_to_phonemes( VERSE_1 ) )