#!/usr/bin/python """ ---------------------------------------------------------- pyCSVoice: Common Sense Voice Recognition Correction Developed by Waseem Daher (wdaher@mit.edu) Software Agents Group MIT Media Lab Spring 2004 ---------------------------------------------------------- See http://www.surguy.net/articles/speechrecognition.xml for code samples and tips on how to use the Microsoft Speech engine in Python ---------------------------------------------------------- How to get started with this code: * Install Python (www.python.org) * Install wxPython (www.wxpython.org) * Install Pythonwin and the Win32 Python utilities (http://starship.python.net/crew/mhammond/) * Install ConceptNet for Python (http://web.media.mit.edu/~hugo/omcsnet/) * Install the Microsoft Speech API (http://www.microsoft.com/speech/download/sdk51/) * Make a Python wrapper to the speech API that Python can use: Do this in Pythonwin by going to Tools | COM Makepy Utility in the menu, and selecting Microsoft Speech from the list * Move this file into the same folder as ConceptNet * Run this code! ---------------------------------------------------------- Last modified 8/1/04 """ # Display extended debug output? DEBUG = 1 def debugprint(string): if DEBUG: print string from wxPython.wx import * import os, sys, math, string import OMCSNetAPI # Feel free to adjust these constants if they improve performance, # these all could definitely use some empirical work NUM_ALTERNATES = 20 # Number of alternates to get from the voice reco engine CONTEXT_AGE_FACTOR = 0.7 # What percent of a word's context score remains when it isn't the most recently spoken word? # List of short words that should be ignored in word promotion shortWords = ['the', 'and', 'but', 'of', 'to', 'in', 'with', 'then', 'a', 'for', 'too', 'who', 'that', 'your'] # Clear the screen when we begin os.system('cls') # Only attempt to do voice recognition when in Windows # In GNU/Linux, just disable it so I can work on the interface if os.name == 'nt': # Import the necessary libraries to make Python talk to the Speech SDK from win32com.client import constants import win32com.client import pythoncom # Initially handle voice recognition events here class VoiceRecoHandler(win32com.client.getevents("SAPI.SpSharedRecoContext")): def OnRecognition(self, StreamNumber, StreamPosition, RecognitionType, Result): newResult = win32com.client.Dispatch(Result) vrAlternates = newResult.Alternates(NUM_ALTERNATES) alternates = [] for alternate in vrAlternates: alternates.append(alternate.PhraseInfo.GetText()) # Send the word and the alternates to the newSpeech method application.mainForm.newSpeech(alternates) else: print "Regrettably, this application requires Microsoft Windows" print "since it uses the Microsoft Speech API." # -- Set up the wxIDs that I use for name in ['MAINFRAME', 'TEXTBOX', 'VRALT', 'CNALT', 'ALTERNATES', 'ADDWORD', 'ADDWORDBUTTON', 'REFRESHBUTTON', 'CLEARBUTTON', 'UNDO']: exec('wxID_%s = wxNewId()' % name) class pyCSVoiceApp(wxFrame): # -- Mostly GUI-related functions: def __init__(self, parent, id, title): wxFrame.__init__(self, parent, id, title, pos = wxDefaultPosition, size = wxSize(830, 520), style = wxDEFAULT_FRAME_STYLE) # -- Let's get a normal-colored background self.SetBackgroundColour(wxNamedColour('WHITE')) self.Refresh() # -- Set up menu bar file = wxMenu() self.addMenuEvent(file, self.menuExit, 'E&xit', 'Quit Common Sense Voice Recognition Correction') help = wxMenu() self.addMenuEvent(help, self.menuAbout, 'About this program...', 'About Common Sense Voice Recognition Correction') menuBar = wxMenuBar() menuBar.Append(file, '&File') menuBar.Append(help, '&Help') self.SetMenuBar(menuBar) # -- Set up main input textbox self.txtMainInput = wxTextCtrl(parent = self, id = wxID_TEXTBOX, value = '', pos = wxPoint(10, 20), size = wxSize(530, 200), style = wxTE_MULTILINE) self.lblMainInput = wxStaticText(self, -1, 'Voice Recognition Results', wxPoint(10,5)) # -- Set up listboxes of alternates self.lstVRAlternates = wxListBox(parent = self, id = wxID_VRALT, pos = wxPoint(10, 250), size = wxSize(250, 200), style = wxLC_LIST) self.lblVRAlternates = wxStaticText(self, -1, 'Voice Recognition Alternates', wxPoint(10,235)) self.lstCNAlternates = wxListBox(parent = self, id = wxID_CNALT, pos = wxPoint(290, 250), size = wxSize(250, 200), style = wxLC_LIST) self.lblCNAlternates = wxStaticText(self, -1, 'Conceptually-related words', wxPoint(290,235)) self.contextList = [] self.lstAlternates = wxListBox(parent = self, id = wxID_ALTERNATES, pos = wxPoint(550, 20), size = wxSize(250, 200), style = wxLC_LIST) self.lblAlternates = wxStaticText(self, -1, 'Alternates', wxPoint(550,5)) self.alternates = [] self.prettyAlternates = [] self.txtAddWords = wxTextCtrl(parent = self, id = wxID_ADDWORD, value = '', pos = wxPoint(550, 250), size = wxSize(250, 20)) self.lblAddWords = wxStaticText(self, -1, 'Add words using keyboard', wxPoint(550,235)) self.btnAddWords = wxButton(self, wxID_ADDWORDBUTTON, 'Add words', wxPoint(550, 275), wxSize(250, 20)) self.btnRefreshWords = wxButton(self, wxID_REFRESHBUTTON, 'Refresh ConceptNet from big textbox', wxPoint(550, 300), wxSize(250, 20)) self.btnClearWords = wxButton(self, wxID_CLEARBUTTON, 'Clear all', wxPoint(550, 325), wxSize(250, 20)) self.btnUndoAdd = wxButton(self, wxID_UNDO, 'Undo!', wxPoint(550, 350), wxSize(250, 20)) # -- Initialize ConceptNet self.conceptNet = OMCSNetAPI.OMCSNetAPI() # -- Event handlers EVT_BUTTON(self, wxID_ADDWORDBUTTON, self.addWordButton) EVT_BUTTON(self, wxID_REFRESHBUTTON, self.refreshCnet) EVT_BUTTON(self, wxID_CLEARBUTTON, self.clearWords) EVT_BUTTON(self, wxID_UNDO, self.undoButtonClick) EVT_LISTBOX_DCLICK(self, wxID_ALTERNATES, self.selectedDifferentAlternate) # -- Set up ConceptNet stuff initially self.oldContextList = [] self.oldText = '' # -- Event handlers def selectedDifferentAlternate(self, event): # Figure out which alternate it was # and pass it to the undoHandler selectedItems = self.lstAlternates.GetSelections() # Presumably only one item is selected - let's operate in that assumption debugprint('Different alternate (' + selectedItems[0] + ') Selected from listbox') self.makeVoiceCorrection(selectedItems[0]) def undoButtonClick(self, event): debugprint('Undo Button clicked') self.undoLastAction() def addWordButton(self, event): debugprint('AddWordButton clicked') newWords = self.txtAddWords.GetValue() self.addAndDisplay(newWords) def clearWords(self, event): debugprint('Clearing words') self.clearConceptNetList() self.txtMainInput.Clear() self.displayNewAlternates() self.lstAlternates.Clear() self.lstVRAlternates.Clear() def refreshCnet(self, event): debugprint('Refreshing conceptnet from textbox') self.clearConceptNetList() words = self.txtMainInput.GetValue() self.addPhraseToCnet(words) self.displayNewAlternates() def menuExit(self, event): debugprint('Exiting...') # Shut down the VR engine nicely here, please # Destroy my window self.Destroy() def menuAbout(self, event): debugprint('AboutWindow opened') wxMessageBox('Common Sense Voice Recognition Correction\n\nWaseem Daher (wdaher@mit.edu)\nSoftware Agents Group\nMIT Media Lab') # -- Non-event handlers def newSpeech(self, alternatesList = []): """ This function gets called when a new word is found """ debugprint('New speech event') # Are any of these command words? word2num = {'first': 1, 'second': 2, 'third': 3, 'fourth': 4, 'fifth': 5, 'sixth': 6, 'seventh': 7, 'eighth' : 8, 'ninth': 9, 'tenth': 10, 'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6, 'seven': 7, 'eight' : 8, 'nine': 9, 'ten': 10,} words = alternatesList[0].lower().split() if words[0] in ['undo', 'undue']: self.undoLastAction() return elif words[0] == 'oops': debugprint('Someone said oops') self.pickFirstRankedAlternate() return # allow 'select nth' to select an alternate elif words[0] == 'select' and words[1] in word2num.keys(): debugprint('Selecting alternate') self.makeVoiceCorrection(word2num[words[1]]) return # No, no special words, let's just list the alternates self.lstVRAlternates.Clear() self.lstVRAlternates.InsertItems(alternatesList, 0) self.vrAlternates = alternatesList self.calculateAlternates() def calculateAlternates(self): """ List alternates """ os.system('cls') self.intelligentAlternateSort() self.lstAlternates.Clear() self.lstAlternates.InsertItems([str(i)+". "+self.prettyAlternates[i] for i in range(0, len(self.prettyAlternates))], 0) # Add the phrase to our contextList, and update accordingly self.addAndDisplay(self.prettyAlternates[0]) for line in self.prettifyAlternates(): print line def intelligentAlternateSort(self): """ Sort alternates """ # Helper function to help sort def contextListSorter(a, b): return cmp(b[1], a[1]) def cleanUp(word): newWord = word.strip().lower() if newWord[-1] in string.punctuation: newWord = newWord[:-1] if newWord[0] in string.punctuation: newWord = newWord[1:] return newWord # Maybe we shouldn't make all the words have the exact same weight -- option self.alternates = [(alternate, 1) for alternate in self.vrAlternates] # Now let's see which ones include good words, and modify their score accordingly for phrase in self.vrAlternates: for word in phrase.split(): if word not in shortWords and len(word) > 2: # Eliminate all short words for now wordsWeHave = [entry[0] for entry in self.contextList] for contextWords in wordsWeHave: if cleanUp(word) in contextWords.split(): # Ok, this word appears in both the contextList and voiceReco -- promote me! currentLocation = self.vrAlternates.index(phrase) locInContextList = wordsWeHave.index(contextWords) # Presumably this adds the score from the contextList to the score of the promoted word print "Promoted '", phrase, "' owing to '", contextWords, "'" self.alternates[currentLocation] = (phrase, self.alternates[currentLocation][1] \ + self.contextList[locInContextList][1]) # Sort me now self.alternates.sort(contextListSorter) # Create prettyAlternates self.prettyAlternates = [word[0] for word in self.alternates] def prettifyAlternates(self): debugprint('Prettifying alternates') output = [] for item in self.alternates: output.append("%s (%d%%)" % (item[0], (item[1] * 100))) return output def makeVoiceCorrection(self, index): debugprint('Making voice correction') # This procedure, called either by a double-click on the alternates box # or a spoken trigger, replaces the last word with the word at index # in self.alternates self.undoLastAction() newWords = self.prettyAlternates[index] self.addAndDisplay(newWords) def addAndDisplay(self, newWords): debugprint('AddAndDisplay: ' + str(newWords)) self.addPhraseToCnet(newWords) self.displayNewAlternates() # Update the big textbox with our nice value self.txtMainInput.SetValue(self.txtMainInput.GetValue() + newWords + " ") # Move the cursor to the end of the textbox self.txtMainInput.SetInsertionPointEnd() def undoLastAction(self): debugprint('undoLastAction') self.contextList = self.oldContextList self.txtMainInput.SetValue(self.oldText) self.displayNewAlternates() def addPhraseToCnet(self, words): debugprint('Adding phrase to cnet: ' + str(words)) self.addWordsToCnet(words.split()) def addWordsToCnet(self, newList): debugprint('Adding words to cnet: ' + str(newList)) # First, back up the old contextList self.oldContextList = self.contextList self.oldText = self.txtMainInput.GetValue() # Now, get the new one for word in newList: self.ageConceptNet() self.updateConceptNetList(word) def ageConceptNet(self): debugprint('Aging conceptnet') # Age all the existing words because we're going to add # new ones in. # Age the words by (1-CONTEXT_AGE_FACTOR) each time... this of course can be changed # depending on what works best self.contextList = [(word[0], word[1] * CONTEXT_AGE_FACTOR) for word in self.contextList] def clearConceptNetList(self): debugprint('clearing conceptnet') self.contextList = [] #self.wordnum = -1 def updateConceptNetList(self, word): debugprint('updating conceptnet with ' + str(word)) # Helper function to help sort def contextListSorter(a, b): return cmp(b[1], a[1]) def cleanUp(word): newWord = word.strip().lower() if newWord[-1] in string.punctuation: newWord = newWord[:-1] if newWord[0] in string.punctuation: newWord = newWord[1:] return newWord debugprint('Trying to get related words for ' + str([cleanUp(word)])) relatedWords = self.conceptNet.get_context([cleanUp(word)]) debugprint('Related words: ' + str(relatedWords)) if relatedWords: for item in relatedWords: # first, do we already have this word? # Since this is a messy data structure, this is going to be annoying wordsWeHave = [entry[0] for entry in self.contextList] if item[0] in wordsWeHave: # already have the word, do something smart here # For now just add their two scores together -- this will give us # dumb, greater than 100, percentages, but so be it. # We can suppress the output of the percentages or rethink how to do this # if this ends up being a problem currentLocation = wordsWeHave.index(item[0]) # Just add the score and replace self.contextList[currentLocation] = (item[0], item[1] + self.contextList[currentLocation][1]) else: self.contextList.append(item) # Sort self.contextList self.contextList.sort(contextListSorter) def displayNewAlternates(self): output = [] for item in self.contextList: output.append("%s (%d%%)" % (item[0], (item[1] * 100))) self.lstCNAlternates.Clear() self.lstCNAlternates.InsertItems(output, 0) def addMenuEvent(self, menu, callbackMethod, label, tooltip, checked=0): id = wxNewId() EVT_MENU(self, id, callbackMethod) item = wxMenuItem(menu, id, label, tooltip, checked) menu.AppendItem(item) return item class MyApp(wxApp): def __init__(self, foo): wxApp.__init__(self, foo) def InitSpeech(self): listener = win32com.client.Dispatch("SAPI.SpSharedRecognizer") self.context = listener.CreateRecoContext() self.grammar = self.context.CreateGrammar() self.grammar.DictationSetState(1) # Enable free-form dictation debugprint('Free-form dictation enabled') def OnInit(self): self.mainForm = pyCSVoiceApp(NULL, -1, 'Common Sense Voice Recognition Correction') self.mainForm.Show(true) if os.name == 'nt': self.InitSpeech() events = VoiceRecoHandler(self.context) self.SetTopWindow(self.mainForm) debugprint('Application initialized') return True if __name__ == '__main__': debugprint('Starting application...') application = MyApp(0) debugprint('Entering main event loop...') application.MainLoop()