gestures

commit 366f1d93427ff56289ded727f3b5c02715a49698
parent fa306ffeb17a0358283f0fa8a0d27b8152b0fca4
Author: Stefan Koch <taikano@lavabit.com>
Date:   Fri,  7 Jun 2013 10:08:23 +0200

added flexible code for new gestures

Diffstat:
A
models.py
 | 
77
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
M
track2.py
 | 
43
+++++++++++++++++++++++++++----------------

2 files changed, 104 insertions(+), 16 deletions(-)
diff --git a/models.py b/models.py
@@ -0,0 +1,77 @@
+import ghmm
+
+UP = 0
+DOWN = 1
+LEFT = 2
+RIGHT = 3
+GESTES_COUNT = 4
+OBSERVATIONS_COUNT = 4
+
+gestures = [
+    ([UP, RIGHT], ['/usr/bin/xdotool', 'getactivewindow', 'windowkill']),
+    ([UP, LEFT], ['notify-send', '"received command"', '"Hooray!"']),
+]
+
+# TODO: Zuordnung von Indizes stimmt noch nicht
+def transition_matrix(gesture):
+    gestes = list(set(gesture))
+    A = [[0 for i in range(GESTES_COUNT)] for j in range(GESTES_COUNT)]
+    # self transitions are high
+    for geste in gestes:
+        A[geste][geste] = 0.7
+
+    first = gestes[0]
+
+    for second in gestes[1:]:
+        # if we have a transition, prob must be high
+        A[first][second] = 0.3
+
+    # ending element has no transitions anymore, so give it one
+    A[gestes[-1]][gestes[-1]] = 1
+
+    return A
+
+def emission_matrix(gesture):
+    gestes = set(gesture)
+    B = [[0 for i in range(OBSERVATIONS_COUNT)] for j in range(GESTES_COUNT)]
+    
+    for geste in gestes:
+        if geste == UP:
+            B[geste] = [0.5, 0.1, 0.2, 0.2]
+        elif geste == DOWN:
+            B[geste] = [0.1, 0.5, 0.2, 0.2]
+        elif geste == LEFT:
+            B[geste] = [0.2, 0.2, 0.5, 0.1]
+        else:
+            B[geste] = [0.2, 0.2, 0.1, 0.5]
+
+    return B
+
+
+# Construct parameters
+models = []
+
+sigma = ghmm.IntegerRange(0, 4)
+for gesture in gestures:
+    # transition matrix
+    A = transition_matrix(gesture[0])
+    B = emission_matrix(gesture[0])
+    pi = [1, 0, 0, 0] # TODO: make flexible depending on first vector
+                # in gesture
+    m = ghmm.HMMFromMatrices(sigma, ghmm.DiscreteDistribution(sigma), A, B, pi)
+    models.append((m, gesture[1]))
+
+#A = [[0.7, 0.3], [0, 1]]
+#B = [[0.6, 0.1, 0.2, 0.2], [.2, .2, .1, .6]]
+#pi = [1, 0]
+#m = ghmm.HMMFromMatrices(sigma, ghmm.DiscreteDistribution(sigma), A, B, pi)
+
+#A2 = [[0.7, 0.3], [0, 1]]
+#B2 = [[0.6, 0.1, 0.2, 0.2], [0.2, .2, .6, .1]]
+#pi2 = [1, 0]
+#m2 = ghmm.HMMFromMatrices(sigma, ghmm.DiscreteDistribution(sigma), A2, B2, pi2)
+
+#models = [
+#    (m, ['/usr/bin/xdotool', 'getactivewindow', 'windowkill']),
+#    (m2, ['notify-send', '"received command"', '"Hooray!"']),
+#]
diff --git a/track2.py b/track2.py
@@ -2,6 +2,13 @@ import cv2
 import math
 import subprocess
 import numpy
+import ghmm
+import models
+
+UP = 0
+DOWN = 1
+LEFT = 2
+RIGHT = 3
 
 path = []
 
@@ -22,17 +29,20 @@ def pointer_pos(img):
 
     return (None, None)
 
-def execute(path):
-    new_path = []
-    last_elem = None
-    for elem in path:
-        if last_elem != elem:
-            new_path.append(elem)
-            last_elem = elem
+def execute(emission_seq, models):
+    max_comm = None
+    max_val = 0
+    for model, command in models:
+        print(model.forward(emission_seq))
+        res = model.forward(emission_seq)
+
+        if res[1][-1] > max_val:
+            max_val = res[1][-1]
+            max_comm = command
+
+    if max_val >= 0.4:
+        subprocess.call(max_comm)
 
-    print new_path
-    if new_path == ['up', 'right']:
-        subprocess.call(['/usr/bin/xdotool', 'getactivewindow', 'windowkill'])
 
 cam = cv2.VideoCapture(0)
  
@@ -40,6 +50,7 @@ winName = "Movement Indicator"
 cv2.namedWindow(winName, cv2.CV_WINDOW_AUTOSIZE)
 
 img = cv2.cvtColor(cam.read()[1], cv2.COLOR_BGR2HSV)
+cv2.imwrite("test.jpg", img)
 img = cv2.inRange(img, (70, 100, 100), (150, 255, 255))
 img = cv2.erode(img, numpy.array([[1,1,1],[1,1,1],[1,1,1]]))
 img = cv2.dilate(img, numpy.array([[1,1,1],[1,1,1],[1,1,1]]), iterations=3)
@@ -56,7 +67,6 @@ while True:
     img = cv2.inRange(img, (70, 50, 50), (150, 255, 255))
     img = cv2.erode(img, numpy.array([[1,1,1,1,1],[1,1,1,1,1],[1,1,1,1,1],[1,1,1,1,1],[1,1,1,1,1]]), iterations=2)
     img = cv2.dilate(img, numpy.array([[1,1,1],[1,1,1],[1,1,1]]), iterations=3)
-
     x1, y1 = pointer_pos(img)
 
     if x1 != None and x0 != None and y1 != None and y0 != None:
@@ -66,19 +76,20 @@ while True:
         if abs(x_delta) > 10 or abs(y_delta) > 10:
             degree = math.atan2(y_delta, x_delta)
             if -0.75 * math.pi <= degree < -0.25 * math.pi:
-                path.append('up')
+                path.append(UP)
             elif -0.25 * math.pi <= degree < 0.25 * math.pi:
-                path.append('left')
+                path.append(LEFT)
             elif 0.25 * math.pi <= degree < 0.75 * math.pi:
-                path.append('down')
+                path.append(DOWN)
             else:
-                path.append('right')
+                path.append(RIGHT)
             print("Appended to")
             print(path)
         else:
             not_changed += 1
     if not_changed > 5:
-        execute(path)
+        if len(path) >= 2:
+            execute(ghmm.EmissionSequence(models.sigma, path), models.models)
         path = []
         not_changed = 0