Minor improvements to readme files and a build script

.dockerignore

@@ -0,0 +1,10 @@
+.git
+
+**/credentials.json
+**/robot_tuner.dat
+**/GoogleNews-vectors-negative300.bin.gz
+**/model.vec
+**/classify_image_graph_def.pb
+**/classifier_candy_sorter*
+**/logs/*
+**/tmp

.gitignore

@@ -95,6 +95,9 @@ ENV/
 
 ### Application
 
+# data
+data/
+
 # Log rotation
 *.log.*
 

README.md

@@ -16,15 +16,24 @@ In the training mode, the system uses Inception model with transfer learning on
 - Controller PC (Linux) and camera: recognizes the candies with Cloud ML, controls the robot arm
 - Robot arm: picks the candy, takes it to the certain location, and drops.
 
+### Candy type
+The robot can handle two types of candy. Small candy that can be picked up, for which it will correcly adjust the gripper to account for rotation etc. It also supports boxed/smooth candy which it picks up using a suction cup.
+
 ## Learning mode
 - Android tablet: shows UI for training process updates
 - Controller PC: runs Inception-v3 + transfer learning on Cloud ML to train a model from scratch, with the camera image
+- More on [learning mode](/setup/demo_script.md)
 
 ## Setting things up
 - Prepare [hardware components](./setup/requirements.md)
 - Prepare [candies](./setup/candidate_candies.md)
-- Set up as directed by [setup/README.md](./setup/README.md)
+- Set up as directed by [setup/README.md](./setup/SETUP_GUIDE.md)
+
+## Troublehooting
+- See [this page](/setup/troubleshooting.md) in case you encounter any errors or unexpected behaviour.
+
+## Troublehooting
+- See [this page](/setup/troubleshooting.md) in case you encounter any errors or unexpected behaviour.
 
 ## Note
-- Currently, the system is not using Cloud Speech API. It uses Web Speech API that shares the same voice recognition backend.
 - For Learning mode it is using Cloud ML training. For Serving mode it is not using Cloud ML prediction.

robot-arm/README.md

@@ -3,10 +3,8 @@ robot-arm (api)
 ===
 
 ## Note
-This software is design to work with Python2.7 under following condition.
+This software is designed to work with Python3.6 under following conditions:
 - This software depends on the following libraries:
- - OpenCV3.2 (need to be compiled from source.)
-  - Refer to [setup/README.md](../setup/README.md) for installation.
 - Environment variables
     ```
     export FLASK_ENV='prd'   # choices are 'prd', 'stg' or 'dev'
@@ -28,7 +26,7 @@ $ sudo systemctl start uwsgi-robot.service
 In this setup senario, the next command is not used. We use nginx+uWSGI instead.
 ```
 # Run app. This requres environment variables.
-$ python2 run_api.py  # Be sure to use python2.7
+$ python3 run_api.py  # Be sure to use python2.7
 ```
 
 ## API example

robot-arm/calibration/converter.py

@@ -22,7 +22,7 @@
 
 
 class CoordinateConverter(object):
-    def __init__(self, from_points, to_points, z_low=-69):
+    def __init__(self, from_points, to_points, z_low=-31, r=0):
         """
         find T that meets x_robot = T x_logical
         """
@@ -39,18 +39,23 @@ def __init__(self, from_points, to_points, z_low=-69):
 
         self.mat_transform = np.dot(self.mat_x_to.T, inv_x)
         self.z_low = z_low
+        self.r_offset = r
 
-    def convert(self, x, y):
+    def convert(self, x, y, r = None):
         xy_trans = AdjustForPictureToRobot()
         x, y = xy_trans.adjust(x, y)
 
         from_vector = np.array([x, y, 1])
         transformed = np.dot(from_vector, self.mat_transform.T)
 
-        return transformed[0], transformed[1]
+        if r is None:
+            return transformed[0], transformed[1]
+        else:
+            adjusted_r = r + self.r_offset
+            return transformed[0], transformed[1], adjusted_r
 
     @classmethod
-    def from_tuning_file(cls, file_='/tmp/robot_tuner.dat', z_low_pad=5):
+    def from_tuning_file(cls, file_, z_low_pad=1):
         tuner_data = []
         with open(file_, 'r') as readfile:
             for line in readfile:
@@ -59,11 +64,16 @@ def from_tuning_file(cls, file_='/tmp/robot_tuner.dat', z_low_pad=5):
                 data = json.loads(line)
                 tuner_data.append(data)
 
-        z_low = sum([d['z'] for d in tuner_data])/3 + z_low_pad
+        # Find z low by averaging the measured z's
+        z_low = sum([d['z'] for d in tuner_data[:3]])/3 + z_low_pad
+        # If there is a fourth line, it contains the r-offset.
+        r = 0 if len(tuner_data) < 4 else tuner_data[3]['r']
+
         return cls(
             [(-0.3, 1.5), (-0.3, -1.5), (0.3, 0)],
             [(tuner_data[0]['x'], tuner_data[0]['y']),
              (tuner_data[1]['x'], tuner_data[1]['y']),
              (tuner_data[2]['x'], tuner_data[2]['y'])],
-            z_low=z_low
+            z_low=z_low,
+            r=r
         )

robot-arm/calibration/example_tune.dat

@@ -1,3 +1,5 @@
-{"x":78.83, "y": 132.45, "z":-70.82}
-{"x":74.41, "y":-145.70, "z":-70.06}
-{"x":175.01, "y": -7.77, "z":-69.91}
+{"x": 172.98753356933594, "y": 176.05148315429688, "z": -35.271888732910156}
+{"x": 181.35813903808594, "y": -169.5199432373047, "z": -30.119842529296875}
+{"x": 246.0970458984375, "y": 6.5959792137146, "z": -33.612144470214844}
+{"r": 7.0}
+{"x": 24.846025466918945, "y": 303.1612243652344, "z": 60.07351303100586}

robot-arm/dobot/alarms.py

@@ -0,0 +1,74 @@
+alarms = {
+    "0x00": {
+        "Category": "Public Alarm",
+        "Name": "Reset Alarm",
+        "SetCondition": " After the system reset, the reset alarm will be set automatically",
+        "ResetCondition": " Protocol instruction is cleared"
+    },
+    "0x01": {
+        "Category": "Public Alarm",
+        "Name": "Undefined Instruction",
+        "SetCondition": "Undifined instruction is received",
+        "ResetCondition": "The protocol instruction is cleared."
+    },
+    "0x02": {
+        "Category": "Public Alarm",
+        "Name": "File System Error",
+        "SetCondition": "The file system errors;",
+        "ResetCondition": "Reset, if the file system initialization is successful, then reset the alarm automatically."
+    },
+    "0x03": {
+        "Category": "Public Alarm",
+        "Name": "Failed Communication between MCU and FPGA",
+        "SetCondition": "The communication between MCU and FPGA is failed when initialization;",
+        "ResetCondition": "Reset, if the communication is successful, then reset the alarm automatically"
+    },
+    "0x04": {
+        "Category": "Public Alarm",
+        "Name": "Angle Sensor Reading Error",
+        "SetCondition": "The angle sensor value can not be read correctly",
+        "ResetCondition": "Power off, power on again, if the angle sensor value can be read correctly, then reset alarm."
+    },
+    "0x11": {
+        "Category": "Planning Alarm",
+        "Name": "Inverse Resolve Alarm",
+        "SetCondition": "The planning target point is not in the robot work space, resulting in the reverse solution error",
+        "ResetCondition": "The protocol instruction is cleared"
+    },
+    "0x12": {
+        "Category": "Planning Alarm",
+        "Name": "Inverse Resolve Limit",
+        "SetCondition": "Inverse resolve of the target point beyond the joint limit value",
+        "ResetCondition": "The protocol instruction is cleared"
+    },
+    "0x13": {
+        "Category": "Planning Alarm",
+        "Name": "Arc Input Parameter Alarm",
+        "SetCondition": "Enter the midpoint of the arc, and the target point can not form an arc",
+        "ResetCondition": "The protocol instruction is cleared"
+    },
+    "0x15": {
+        "Category": "Planning Alarm",
+        "Name": "JUMP Parameter Error",
+        "SetCondition": "The planning starting point or the target point height above the set maximum JUMP height",
+        "ResetCondition": "The protocol instruction is cleared"
+    },
+    "0x21": {
+        "Category": "Kinematic Alarm",
+        "Name": "Inverse Resolve Alarm",
+        "SetCondition": "The movement process beyond the robot work space resulting in the inverse solution error",
+        "ResetCondition": "The protocol instruction is cleared"
+    },
+    "0x22": {
+        "Category": "Kinematic Alarm",
+        "Name": "Inverse Resolve Limit",
+        "SetCondition": "The inverse solution of the motion exceeds the Joint limit value",
+        "ResetCondition": "The protocol instruction is cleared"
+    },
+    "0x44": {
+        "Category": "Limit Alarm",
+        "Name": "Joint 3 Positive Limit Alarm",
+        "SetCondition": "Joint 3 moves in the positive direction to the limit area",
+        "ResetCondition": "Protocol command reset the alarm manually; reset the alarm in reverse exit limit area automatically"
+    }
+}

robot-arm/dobot/client.py

@@ -18,7 +18,9 @@
 import logging
 import time
 
+
 from dobot import command
+from dobot import alarms
 from dobot.errors import TimeoutError
 from dobot.serial import SerialCommunicator
 
@@ -32,6 +34,7 @@ def __init__(self, port, baudrate, read_timeout_sec=10, serial_timeout_sec=5):
             port, baudrate, read_timeout_sec=read_timeout_sec, serial_timeout_sec=serial_timeout_sec
         )
 
+
     def get_pose(self):
         return self.serial.call(command.GetPose())
 
@@ -51,40 +54,78 @@ def count_queued_command(self):
         result = self.serial.call(command.GetQueuedCmdLeftSpace())
         return DOBOT_QUEUE_SIZE - result['leftSpace']
 
+    def get_alarms_state(self, discard_reset_alarm=True):
+        # Get the alarms from the Dobot
+        result = self.serial.call(command.GetAlarmsState())
+        # Process alarm data into a string with bit flags
+        alarm_string = "".join(["{0:0>8b}".format(x)[::-1] for x in list(result)[4:-1]])  # Get only the alarm values
+        # Traverse the bit flags and extract the alarm IDs
+        alarm_ids = ["0x" + "{0:0>2X}".format(index) for index, value in
+                     enumerate(alarm_string[int(discard_reset_alarm):]) if value == "1"]
+        # Lookup alarm ids to get the information about the alarms
+        dobot_alarms = [alarms.alarms.get(alarm_id) if alarms.alarms.get(alarm_id) else alarm_id
+                        for alarm_id in alarm_ids]
+        return dobot_alarms
+
+    def clear_alarms_state(self):
+        return self.serial.call(command.ClearAllAlarmsState())
+
     def initialize(self):
         self.serial.call(command.ClearAllAlarmsState())
         self.serial.call(command.SetQueuedCmdClear())
         self.adjust_z(0)
         self.serial.call(command.SetHomeCmd())
 
-    def move(self, x, y, z, velocity=200, accel=200, jump=True):
+    def move(self, x, y, z, r=0, velocity=200, accel=200, jump=True):
         self.serial.call(command.SetPTPJointParams(velocity, velocity, velocity, velocity, accel, accel, accel, accel))
 
         mode = 0
         if not jump:
             mode = 1
 
-        self.serial.call(command.SetPTPCmd(mode, x, y, z, 0))
+        self.serial.call(command.SetPTPCmd(mode, x, y, z, r))
 
-    def linear_move(self, x, y, z, velocity=200, accel=200):
+    def linear_move(self, x, y, z, r=0, velocity=200, accel=200):
         self.serial.call(command.SetPTPCoordinateParams(velocity, velocity, accel, accel))
-        self.serial.call(command.SetPTPCmd(2, x, y, z, 0))
+        self.serial.call(command.SetPTPCmd(2, x, y, z, r))
 
     def pickup(self, x, y, z_low=0, z_high=100, sleep_sec=1, velocity=200, accel=100, num_trials=1):
-        self.move(x, y, z_high, velocity, accel)
+        self.move(x, y, z_high, 0,  velocity, accel)
         self.pump(1)
         for i in range(num_trials):
-            self.linear_move(x, y, z_low, velocity, accel)
+            self.linear_move(x, y, z_low, 0, velocity, accel)
+            time.sleep(sleep_sec)
+            self.linear_move(x, y, z_high, 0, velocity, accel)
+
+    def pickup_gripper(self, x, y, r, z_low=0, z_high=100, sleep_sec=0.5, velocity=200, accel=100, num_trials=1):
+        self.grip(0)
+        self.move(x, y, z_high, r,  velocity, accel)
+        for i in range(num_trials):
+            self.linear_move(x, y, z_low, r, velocity, accel)
+            time.sleep(1)
+            self.grip(1)
+            self.wait(120)
             time.sleep(sleep_sec)
-            self.linear_move(x, y, z_high, velocity, accel)
+            self.linear_move(x, y, z_high, r, velocity, accel)
 
     def adjust_z(self, z):
         self.wait()
         pose = self.serial.call(command.GetPose())
-        self.linear_move(pose['x'], pose['y'], z)
+        self.linear_move(pose['x'], pose['y'], z, pose['r'])
+
+    def adjust_r(self, r):
+        self.wait()
+        pose = self.serial.call(command.GetPose())
+        self.linear_move(pose['x'], pose['y'], pose['z'], r)
 
     def pump(self, on):
         self.serial.call(command.SetEndEffectorSuctionCup(1, on))
 
+    def grip(self, grip):
+        self.serial.call(command.SetEndEffectorGripper(1, grip))
+
+    def stepper_motor(self, index, enabled, reverse=False):
+        self.serial.call(command.SetEMotor(index, enabled, -10000 if reverse else 10000))
+
     def close(self):
         self.serial.close()

robot-arm/dobot/command.py

@@ -102,7 +102,7 @@ def parse_to_dict(self, packet):
 
 
 def checksum(payload):
-    return 0xFF & (0x100 - (sum([ord(c) for c in payload])))
+    return 0xFF & (0x100 - (sum([c for c in payload])))
 
 
 def ctrl(rw, is_queued):
@@ -129,7 +129,9 @@ def __init__(self):
 
 class GetAlarmsState(Command):
     def __init__(self):
-        super(GetAlarmsState, self).__init__(20, 0, 0)
+        super(GetAlarmsState, self).__init__(
+            20, 0, 0, '16B'
+        )
 
 
 class ClearAllAlarmsState(Command):
@@ -199,6 +201,29 @@ def __init__(self, is_ctrl_enabled, is_sucked):
         self.params = struct.pack('<BB', is_ctrl_enabled, is_sucked)
 
 
+class SetEndEffectorGripper(Command):
+    def __init__(self, is_ctrl_enabled, is_gripped):
+        super(SetEndEffectorGripper, self).__init__(63, 1, 1)
+        self.params = struct.pack('<BB', is_ctrl_enabled, is_gripped)
+
+
+"""
+Enable stepper motor. While protocal says that speed is a float, it is an int. 
+10000 will give approx 62mm/s, which is below 100mm/s (max) and not too far from
+50-60mm/s which has been recommended by Dobot support. 
+"""
+class SetEMotor(Command):
+    def __init__(self, index, isEnabled, speed, is_queued=True):
+        params_fmt = None
+        params_name = None
+        if is_queued:
+            params_fmt = 'Q'
+            params_name = ['queuedCmdIndex']
+
+        super(SetEMotor, self).__init__(135, 1, 1 if is_queued else 0, params_fmt, params_name)
+        self.params = struct.pack('<BBi', index, isEnabled, speed)
+
+
 class SetQueuedCmdStartExec(Command):
     def __init__(self):
         super(SetQueuedCmdStartExec, self).__init__(240, 1, 0)

robot-arm/dobot/serial.py

@@ -27,7 +27,7 @@
 
 
 def dump_hex(s):
-    return ":".join("{:02x}".format(ord(c)) for c in s)
+    return ":".join("{:02x}".format(c) for c in s)
 
 
 class SerialCommunicator(object):

robot-arm/dobot/utils.py

@@ -33,9 +33,10 @@ def detect_dobot_port(baudrate):
 
 def detect_ports(baudrate):
     port_prefix = '/dev/ttyUSB'
+    all_ports = [port_prefix + str(i) for i in range(5)]
+    all_ports.extend(["COM" + str(i) for i in range(5)])
     found_ports = []
-    for i in range(5):
-        port = port_prefix + str(i)
+    for port in all_ports:
         logger.debug('scanning port: {}'.format(port))
         try:
             s = serial.Serial(port, baudrate, timeout=1)
@@ -50,7 +51,7 @@ def dobot_is_on_port(port, baudrate):
     try:
         ser = SerialCommunicator(port, baudrate)
         ser.call(command.GetPose())
-    except IOError as e:
+    except Exception as e:
         print(e)
         return False
     return True