final_version >,<

camera.py

@@ -47,13 +47,17 @@ def get_coordinate(self):
             else: aim_instrument='fork'
         else:
             aim_instrument=max(instrument_dict,key=instrument_dict.get) #若两个工具检测到的次数相同，返回第一个
-        temp=np.array(aim_data[aim_data['label'].isin([aim_instrument])])[-1]
-        aim_coordinate=temp[1:]
-        if aim_instrument=='toothbrush' or aim_instrument=='tie' or aim_instrument=='scissors':
-            aim_instrument='knife'
+        try:
+            temp=np.array(aim_data[aim_data['label'].isin([aim_instrument])])[-1]
+            aim_coordinate = temp[1:]
+            if aim_instrument=='toothbrush' or aim_instrument=='tie' or aim_instrument=='scissors':
+                aim_instrument='knife'
+            #米转厘米
+                aim_coordinate=100*aim_coordinate
+        except:
+            aim_instrument='not found'
+            aim_coordinate=[0,0,0]
 
-        #米转厘米
-        aim_coordinate=100*aim_coordinate
         return aim_instrument,aim_coordinate
 
 

main.py

@@ -34,20 +34,23 @@ def run(self):
                 return 
 
     def pipeline_from_hand(self):
-        # class_name,real_coordinate=self.camera.get_coordinate()
-        # # aim_coordinate=self.console.aihuan_algrithm(real_coordinate)
-        # aim_coordinate=list(real_coordinate)
-        # #假设直接抓取并能成功
-        # print(aim_coordinate)
-        # print(class_name)
-        # self.console.to_xyz(aim_coordinate)
-        # self.console.claw_close()
-        # self.console.place_instrument(class_name)
-        #
-        self.console.claw_open()
-        self.console.to_xyz([25,20,10])
+        class_name,real_coordinate=self.camera.get_coordinate()
+        # aim_coordinate=self.console.aihuan_algrithm(real_coordinate)
+        aim_coordinate=list(real_coordinate)
+        if class_name=='not found':
+            self.microphone.feedback('waiting')
+            self.microphone.feedback('say_again')
+        #假设直接抓取并能成功
+        print(aim_coordinate)
+        print(class_name)
+        self.console.to_xyz(aim_coordinate)
         self.console.claw_close()
-        self.console.place_instrument('fork')
+        self.console.place_instrument(class_name)
+        # self.camera.refresh_obj_file()
+        # self.console.claw_open()
+        # self.console.to_xyz([25,20,10])
+        # self.console.claw_close()
+        # self.console.place_instrument('fork')
 
     def pipeline_to_hand(self,class_name):
         self.console.pick_instrument(class_name)

microphone.py

@@ -11,7 +11,7 @@ def __init__(self):
         self.r=sr.Recognizer()
         self.mic=sr.Microphone(device_index=1)
         self.mode='online'
-        self.ORDER_DICT=['here you are','come here']
+        self.ORDER_DICT=['here you are','come here','you are','are']
         self.INSTRUMENTS_DICT=['knife', 'fork', 'spoon']
         self.KEYWORDS=[['I want a knife',1.0],['I want a fork',1.0],['I want a spoon',1.0],['terminate the programme',1.0]]
         self.p = pyaudio.PyAudio() # instantiate PyAudio (1)

yolo3/utils.py

@@ -8,20 +8,27 @@
 
 def coordinate_transform(a3, b3, c3):
     # 此函数直接从pixel坐标系直接转换为以机械臂为坐标系的坐标
-    [a1, b1] = [298, 144]
-    [a2, b2] = [315, 127]
-    [a4, b4] = [336, 145]
-    [x1, y1] = [1, 0]
-    [x2, y2] = [0, 4]
-    [x4, y4] = [4, 5]
-    camera_height = 74  #
+    [a1, b1] = [362, 167]
+    [a2, b2] = [458, 101]
+    [a4, b4] = [361, 103]
+    [x1, y1] = [-10, 30]
+    [x2, y2] = [-25, 20]
+    [x4, y4] = [-10, 20]
+    camera_height = 0.76  #
     k1 = ((b2-b4)*(a3-a4)-(b3-b4)*(a2-a4))/((a1-a4)*(b2-b4)-(a2-a4)*(b1-b4))
     k2 = ((b1-b4)*(a3-a4)-(a1-a4)*(b3-b4))/((a2-a4)*(b1-b4)-(b2-b4)*(a1-a4))
-    x3 = k1*(x1-x4)+k2*(x2-x4)+x4
-    y3 = k1*(y1-y4)+k2*(y2-y4)+y4
+    x3 = (k1*(x1-x4)+k2*(x2-x4)+x4)/100
+    y3 = (k1*(y1-y4)+k2*(y2-y4)+y4)/100
     z3 = camera_height - c3
+    if y3>0.3:
+        y3=0.3
+    if z3<0.003:
+        z3=0.03
     print(x3, y3, z3)
-    return [x3, y3, z3]
+
+    position=[x3, y3, z3]
+
+    return position
 
 def trans_camera2arm_base(point):
     '''