From c935c8a1f8c12c46bba97e89df1c1ed38d068aeb Mon Sep 17 00:00:00 2001 From: Grigory Sharov Date: Tue, 18 Mar 2025 21:50:55 +0000 Subject: [PATCH 01/11] add old plugin code --- pytemscript/plugins/gatan_sem_socket.py | 713 ++++++++++++++++++++++++ 1 file changed, 713 insertions(+) create mode 100644 pytemscript/plugins/gatan_sem_socket.py diff --git a/pytemscript/plugins/gatan_sem_socket.py b/pytemscript/plugins/gatan_sem_socket.py new file mode 100644 index 0000000..b3f9477 --- /dev/null +++ b/pytemscript/plugins/gatan_sem_socket.py @@ -0,0 +1,713 @@ +""" +The code below is a modified version of: +https://github.com/instamatic-dev/instamatic/blob/main/src/instamatic/camera/gatansocket3.py + +BSD 3-Clause License + +Copyright (c) 2021, Stef Smeets +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The script adapted from [Leginon](https://emg.nysbc.org//projects/leginon/wiki/Leginon_Homepage). Leginon is licenced under the Apache License, Version 2.0. The code (`gatansocket3.py`) was converted from Python2.7 to Python3.6+ from [here](https://github.com/leginon-org/leginon/blob/myami-python3/pyscope/gatansocket.py). + +It needs the SERIALEMCCD plugin to be installed in DigitalMicrograph. The instructions can be found [here](https://bio3d.colorado.edu/SerialEM/hlp/html/setting_up_serialem.htm). + +- On the computer running DM, define an environment variable SERIALEMCCD_PORT=48890 +- [Optional] Set an environment variable SERIALEMCCD_DEBUG=1 or 2 + +""" + +import os +import socket +import logging +import numpy as np + +# enum function codes as in SocketPathway.cpp +# need to match exactly both in number and order +enum_gs = [ + 'GS_ExecuteScript', + 'GS_SetDebugMode', + 'GS_SetDMVersion', + 'GS_SetCurrentCamera', + 'GS_QueueScript', + 'GS_GetAcquiredImage', + 'GS_GetDarkReference', + 'GS_GetGainReference', + 'GS_SelectCamera', + 'GS_SetReadMode', + 'GS_GetNumberOfCameras', + 'GS_IsCameraInserted', + 'GS_InsertCamera', + 'GS_GetDMVersion', + 'GS_GetDMCapabilities', + 'GS_SetShutterNormallyClosed', + 'GS_SetNoDMSettling', + 'GS_GetDSProperties', + 'GS_AcquireDSImage', + 'GS_ReturnDSChannel', + 'GS_StopDSAcquisition', + 'GS_CheckReferenceTime', + 'GS_SetK2Parameters', + 'GS_ChunkHandshake', + 'GS_SetupFileSaving', + 'GS_GetFileSaveResult', + 'GS_SetupFileSaving2', + 'GS_GetDefectList', + 'GS_SetK2Parameters2', + 'GS_StopContinuousCamera', + 'GS_GetPluginVersion', + 'GS_GetLastError', + 'GS_FreeK2GainReference', + 'GS_IsGpuAvailable', + 'GS_SetupFrameAligning', + 'GS_FrameAlignResults', + 'GS_ReturnDeferredSum', + 'GS_MakeAlignComFile', + 'GS_WaitUntilReady', + 'GS_GetLastDoseRate', + 'GS_SaveFrameMdoc', + 'GS_GetDMVersionAndBuild', + 'GS_GetTiltSumProperties', +] +# lookup table of function name to function code, starting with 1 +enum_gs = {x: y for (y, x) in enumerate(enum_gs, 1)} + +# C "long" -> numpy "int_" +ARGS_BUFFER_SIZE = 1024 +MAX_LONG_ARGS = 16 +MAX_DBL_ARGS = 8 +MAX_BOOL_ARGS = 8 +sArgsBuffer = np.zeros(ARGS_BUFFER_SIZE, dtype=np.byte) + + +class Message: + """Information packet to send and receive on the socket. + + Initialize with the sequences of args (longs, bools, doubles) and + optional long array. + """ + + def __init__(self, longargs=[], boolargs=[], dblargs=[], longarray=[]): + # Strings are packaged as long array using np.frombuffer(buffer,np.int_) + # and can be converted back with longarray.tostring() + # add final longarg with size of the longarray + if longarray: + longargs = list(longargs) + longargs.append(len(longarray)) + + self.dtype = [ + ('size', np.intc), + ('longargs', np.int_, (len(longargs),)), + ('boolargs', np.int32, (len(boolargs),)), + ('dblargs', np.double, (len(dblargs),)), + ('longarray', np.int_, (len(longarray),)), + ] + self.array = np.zeros((), dtype=self.dtype) + self.array['size'] = self.array.data.itemsize + self.array['longargs'] = longargs + self.array['boolargs'] = boolargs + self.array['dblargs'] = dblargs + self.array['longarray'] = longarray + + # create numpy arrays for the args and array + # self.longargs = np.asarray(longargs, dtype=np.int_) + # self.dblargs = np.asarray(dblargs, dtype=np.double) + # self.boolargs = np.asarray(boolargs, dtype=np.int32) + # self.longarray = np.asarray(longarray, dtype=np.int_) + + def pack(self): + """Serialize the data.""" + data_size = self.array.data.itemsize + if self.array.data.itemsize > ARGS_BUFFER_SIZE: + raise RuntimeError(f'Message packet size {data_size} is larger than maximum {ARGS_BUFFER_SIZE}') + return self.array.data + + def unpack(self, buf): + """unpack buffer into our data structure.""" + self.array = np.frombuffer(buf, dtype=self.dtype)[0] + + +def logwrap(func): + """Decorator for socket send and recv calls, so they can make log.""" + def newfunc(*args, **kwargs): + logging.debug(f'{func}\t{args}\t{kwargs}') + try: + result = func(*args, **kwargs) + except Exception as ex: + logging.debug(f'EXCEPTION: {ex}') + raise + return result + return newfunc + + +class GatanSocket: + def __init__(self, host, port): + self.host = host + self.port = os.environ.get('SERIALEMCCD_PORT', port) + self.debug = os.environ.get('SERIALEMCCD_DEBUG', 0) + if self.debug: + logging.debug('GatanServerIP =', self.host) + logging.debug('SERIALEMCCD_PORT = GatanServerPort =', self.port) + logging.debug('SERIALEMCCD_DEBUG =', self.debug) + + self.save_frames = False + self.num_grab_sum = 0 + self.connect() + + self.script_functions = [ + ('AFGetSlitState', 'GetEnergyFilter'), + ('AFSetSlitState', 'SetEnergyFilter'), + ('AFGetSlitWidth', 'GetEnergyFilterWidth'), + ('AFSetSlitWidth', 'SetEnergyFilterWidth'), + ('AFDoAlignZeroLoss', 'AlignEnergyFilterZeroLossPeak'), + ('IFCGetSlitState', 'GetEnergyFilter'), + ('IFCSetSlitState', 'SetEnergyFilter'), + ('IFCGetSlitWidth', 'GetEnergyFilterWidth'), + ('IFCSetSlitWidth', 'SetEnergyFilterWidth'), + ('IFCDoAlignZeroLoss', 'AlignEnergyFilterZeroLossPeak'), + ('IFGetSlitIn', 'GetEnergyFilter'), + ('IFSetSlitIn', 'SetEnergyFilter'), + ('IFGetEnergyLoss', 'GetEnergyFilterOffset'), + ('IFSetEnergyLoss', 'SetEnergyFilterOffset'), + ('IFGetSlitWidth', 'GetEnergyFilterWidth'), + ('IFSetSlitWidth', 'SetEnergyFilterWidth'), + ('GT_CenterZLP', 'AlignEnergyFilterZeroLossPeak'), + ] + self.filter_functions = {} + for name, method_name in self.script_functions: + hasScriptFunction = self.hasScriptFunction(name) + if self.hasScriptFunction(name): + self.filter_functions[method_name] = name + if self.debug: + logging.debug(name, method_name, hasScriptFunction) + if ('SetEnergyFilter' in self.filter_functions.keys() and + self.filter_functions['SetEnergyFilter'] == 'IFSetSlitIn'): + self.wait_for_filter = 'IFWaitForFilter();' + else: + self.wait_for_filter = '' + + def hasScriptFunction(self, name): + script = f'if ( DoesFunctionExist("{name}") ) {{ Exit(1.0); }} else {{ Exit(-1.0); }}' + result = self.ExecuteGetDoubleScript(script) + return result > 0.0 + + def connect(self): + # recommended by Gatan to use localhost IP to avoid using tcp + self.sock = socket.create_connection(('127.0.0.1', self.port)) + + def disconnect(self): + self.sock.shutdown(socket.SHUT_RDWR) + self.sock.close() + + def reconnect(self): + self.disconnect() + self.connect() + + @logwrap + def send_data(self, data): + return self.sock.sendall(data) + + @logwrap + def recv_data(self, n): + return self.sock.recv(n) + + def ExchangeMessages(self, message_send, message_recv=None): + self.send_data(message_send.pack()) + + if message_recv is None: + return + recv_buffer = message_recv.pack() + recv_len = recv_buffer.itemsize + + total_recv = 0 + parts = [] + while total_recv < recv_len: + remain = recv_len - total_recv + new_recv = self.recv_data(remain) + parts.append(new_recv) + total_recv += len(new_recv) + buf = b''.join(parts) + message_recv.unpack(buf) + # log the error code from received message + sendargs = message_send.array['longargs'] + recvargs = message_recv.array['longargs'] + logging.debug(f'Func: {sendargs[0]}, Code: {recvargs[0]}') + + def GetFunction(self, funcName, rlongargs=[], rboolargs=[], rdblargs=[]): + """ Common function that only receives data. """ + funcCode = enum_gs[funcName] + message_send = Message(longargs=(funcCode,)) + message_recv = Message(rlongargs, rboolargs, rdblargs) + self.ExchangeMessages(message_send, message_recv) + return message_recv + + def SetFunction(self, funcName, slongargs=[], sboolargs=[], sdblargs=[]): + """ Common function that only sends data. """ + funcCode = enum_gs[funcName] + message_send = Message(longargs=(funcCode, slongargs, sboolargs, sdblargs)) + message_recv = Message(longargs=(0,)) + self.ExchangeMessages(message_send, message_recv) + + def ExecuteSendCameraObjectionFunction(self, function_name, camera_id=0): + # first longargs is error code. Error if > 0 + return self.ExecuteGetLongCameraObjectFunction(function_name, camera_id) + + def ExecuteGetLongCameraObjectFunction(self, function_name, camera_id=0): + """Execute DM script function that requires camera object as input and + output one long integer.""" + recv_longargs_init = (0,) + result = self.ExecuteCameraObjectFunction(function_name, camera_id, + recv_longargs_init=recv_longargs_init) + if result is False: + return 1 + return result.array['longargs'][0] + + def ExecuteGetDoubleCameraObjectFunction(self, function_name, camera_id=0): + """Execute DM script function that requires camera object as input and + output double floating point number.""" + recv_dblargs_init = (0,) + result = self.ExecuteCameraObjectFunction(function_name, camera_id, + recv_dblargs_init=recv_dblargs_init) + if result is False: + return -999.0 + return result.array['dblargs'][0] + + def ExecuteCameraObjectFunction(self, function_name, camera_id=0, recv_longargs_init=(0,), + recv_dblargs_init=(0.0,), recv_longarray_init=[]): + """Execute DM script function that requires camera object as input.""" + if not self.hasScriptFunction(function_name): + # unsuccessful + return False + fullcommand = (f'Object manager = CM_GetCameraManager();\n' + f'Object cameraList = CM_GetCameras(manager);\n' + f'Object camera = ObjectAt(cameraList,{camera_id});\n' + f'{function_name}(camera);\n') + result = self.ExecuteScript(fullcommand, camera_id, recv_longargs_init, + recv_dblargs_init, recv_longarray_init) + return result + + def ExecuteSendScript(self, command_line, select_camera=0): + recv_longargs_init = (0,) + result = self.ExecuteScript(command_line, select_camera, recv_longargs_init) + # first longargs is error code. Error if > 0 + return result.array['longargs'][0] + + def ExecuteGetLongScript(self, command_line, select_camera=0): + """Execute DM script and return the result as integer.""" + # SerialEMCCD DM TemplatePlugIn::ExecuteScript retval is a double + return int(self.ExecuteGetDoubleScript(command_line, select_camera)) + + def ExecuteGetDoubleScript(self, command_line, select_camera=0): + """Execute DM script that gets one double float number.""" + recv_dblargs_init = (0.0,) + result = self.ExecuteScript(command_line, select_camera, recv_dblargs_init=recv_dblargs_init) + return result.array['dblargs'][0] + + def ExecuteScript(self, command_line, select_camera=0, recv_longargs_init=(0,), + recv_dblargs_init=(0.0,), recv_longarray_init=[]): + funcCode = enum_gs['GS_ExecuteScript'] + cmd_str = command_line + '\0' + extra = len(cmd_str) % 4 + if extra: + npad = 4 - extra + cmd_str = cmd_str + (npad) * '\0' + # send the command string as 1D longarray + longarray = np.frombuffer(cmd_str.encode(), dtype=np.int_) + # logging.debug(longaray) + message_send = Message(longargs=(funcCode,), boolargs=(select_camera,), longarray=longarray) + message_recv = Message(longargs=recv_longargs_init, dblargs=recv_dblargs_init, + longarray=recv_longarray_init) + self.ExchangeMessages(message_send, message_recv) + return message_recv + + def RunScript(self, fn: str, background: bool = False): + """Run a DM script. + + fn: str + Path to the script to run + background: bool + Prepend `// $BACKGROUND$` to run the script in the background + and make it non-blocking. + """ + + bkg = r'// $BACKGROUND$\n\n' + + with open(fn, 'r') as f: + cmd_str = ''.join(f.readlines()) + + if background: + cmd_str = bkg + cmd_str + + return self.ExecuteScript(cmd_str) + + +class SocketFuncs(GatanSocket): + def __init__(self, host='127.0.0.1', port=48890): + super().__init__(host, port) + +# ---------- DM functions ----------------------------------------------------- + + def GetDMVersion(self): + message_recv = self.GetFunction('GS_GetDMVersion', + rlongargs=(0, 0)) + result = message_recv.array['longargs'][1] + return result + + def GetDMVersionAndBuild(self): + message_recv = self.GetFunction('GS_GetDMVersionAndBuild', + rlongargs=(0, 0, 0)) + result = message_recv.array['longargs'] + return result[0], result[1] + + def GetDMCapabilities(self): + message_recv = self.GetFunction('GS_GetDMCapabilities', + rlongargs=(0,), + rboolargs=(0, 0, 0)) + result = message_recv.array['boolargs'] + # canSelectShutter, canSetSettling, openShutterWorks + return list(map(bool, result)) + + def GetPluginVersion(self): + message_recv = self.GetFunction('GS_GetPluginVersion', + rlongargs=(0, 0)) + result = message_recv.array['longargs'][1] + return result + + def GetLastError(self): + message_recv = self.GetFunction('GS_GetLastError', + rlongargs=(0, 0)) + result = message_recv.array['longargs'][1] + return result + + def SetDebugMode(self, mode): + self.SetFunction('GS_SetDebugMode', slongargs=(mode,)) + +# ---------- Camera functions ------------------------------------------------- + + def GetNumberOfCameras(self): + message_recv = self.GetFunction('GS_GetNumberOfCameras', + rlongargs=(0, 0)) + result = message_recv.array['longargs'][1] + return result + + def SetCurrentCamera(self, camera): + self.SetFunction('GS_SetCurrentCamera', slongargs=(camera,)) + + def SelectCamera(self, camera): + self.SetFunction('GS_SelectCamera', + slongargs=(camera,)) + + def IsCameraInserted(self, camera): + funcCode = enum_gs['GS_IsCameraInserted'] + message_send = Message(longargs=(funcCode, camera)) + message_recv = Message(longargs=(0,), boolargs=(0,)) + self.ExchangeMessages(message_send, message_recv) + result = bool(message_recv.array['boolargs'][0]) + return result + + def InsertCamera(self, camera, state): + self.SetFunction('GS_InsertCamera', + slongargs=(camera,), + sboolargs=(state,)) + + def SetReadMode(self, mode=-1, scaling=1.0): + """ + Set the read mode and the scaling factor + For K2, pass 0, 1, or 2 for linear, counting, super-res + For K3, pass 3 or 4 for linear or super-res: this is THE signal that it is a K3 + For camera not needing read mode, pass -1 + For OneView, pass -3 for regular imaging or -2 for diffraction + For K3, the offset to be subtracted for linear mode must be supplied with the scaling. + The offset is supposed to be 8192 per frame + The offset per ms is thus nominally (8192 per frame) / (1.502 frames per ms) + pass scaling = trueScaling + 10 * nearestInt(offsetPerMs) + """ + self.SetFunction('GS_SetReadMode', + slongargs=(mode,), + sdblargs=(scaling,)) + + def SetShutterNormallyClosed(self, camera, shutter): + self.SetFunction('GS_SetShutterNormallyClosed', + slongargs=(camera, shutter,)) + + def GetLastDoseRate(self): + message_recv = self.GetFunction('GS_GetLastDoseRate', + rlongargs=(0,), + rdblargs=(0,)) + result = float(message_recv.array['dblargs']) + return result + + @logwrap + def SetK2Parameters(self, readMode, scaling, hardwareProc, doseFrac, + frameTime, alignFrames, saveFrames, filt='', useCds=False): + funcCode = enum_gs['GS_SetK2Parameters'] + + # rotation and flip for non-frame saving image. It is the same definition + # as in SetFileSaving2 + # if set to 0, it takes what GMS has.self.save_frames = saveFrames + rotationFlip = 0 + + # flags + flags = 0 + flags += int(useCds) * 2 ** 6 + reducedSizes = 0 + fullSizes = 0 + + # filter name + filt_str = filt + '\0' + extra = len(filt_str) % 4 + if extra: + npad = 4 - extra + filt_str = filt_str + npad * '\0' + longarray = np.frombuffer(filt_str.encode(), dtype=np.int_) + + longs = [ + funcCode, + readMode, + hardwareProc, + rotationFlip, + flags + ] + bools = [ + doseFrac, + alignFrames, + saveFrames, + ] + doubles = [ + scaling, + frameTime, + reducedSizes, + fullSizes, + 0.0, # dummy3 + 0.0, # dummy4 + ] + + message_send = Message(longargs=longs, boolargs=bools, + dblargs=doubles, longarray=longarray) + message_recv = Message(longargs=(0,)) # just return code + self.ExchangeMessages(message_send, message_recv) + + def setNumGrabSum(self, earlyReturnFrameCount, earlyReturnRamGrabs): + # pack RamGrabs and earlyReturnFrameCount in one double + self.num_grab_sum = (2**16) * earlyReturnRamGrabs + earlyReturnFrameCount + + def getNumGrabSum(self): + return self.num_grab_sum + + @logwrap + def SetupFileSaving(self, rotationFlip, dirname, rootname, filePerImage, + doEarlyReturn, earlyReturnFrameCount=0, earlyReturnRamGrabs=0, + lzwtiff=False): + pixelSize = 1.0 + self.setNumGrabSum(earlyReturnFrameCount, earlyReturnRamGrabs) + if self.save_frames and (doEarlyReturn or lzwtiff): + # early return flag + flag = 128 * int(doEarlyReturn) + 8 * int(lzwtiff) + numGrabSum = self.getNumGrabSum() + # set values to pass + longs = [enum_gs['GS_SetupFileSaving2'], rotationFlip, flag] + dbls = [pixelSize, numGrabSum, 0., 0., 0.] + else: + longs = [enum_gs['GS_SetupFileSaving'], rotationFlip] + dbls = [pixelSize] + bools = [filePerImage] + names_str = dirname + '\0' + rootname + '\0' + extra = len(names_str) % 4 + if extra: + npad = 4 - extra + names_str = names_str + npad * '\0' + longarray = np.frombuffer(names_str.encode(), dtype=np.int_) + message_send = Message(longargs=longs, boolargs=bools, + dblargs=dbls, longarray=longarray) + message_recv = Message(longargs=(0, 0)) + self.ExchangeMessages(message_send, message_recv) + + def StopDSAcquisition(self): + message_recv = self.GetFunction('GS_StopDSAcquisition', + rlongargs=(0, )) + + def StopContinuousCamera(self): + message_recv = self.GetFunction('GS_StopContinuousCamera', + rlongargs=(0, )) + + def GetFileSaveResult(self): + message_recv = self.GetFunction('GS_GetFileSaveResult', + rlongargs=(0, 0, 0)) + result = message_recv.array['longargs'] + # result = numSaved, error + return result[1] + + def SetNoDMSettling(self, value): + self.SetFunction('GS_SetNoDMSettling', + slongargs=(value,)) + + def WaitUntilReady(self, value): + self.SetFunction('GS_WaitUntilReady', + slongargs=(value,)) + + @logwrap + def GetImage(self, processing, height, width, binning, top, + left, bottom, right, exposure, corrections, + shutter=0, shutterDelay=0.): + """ + :param processing: dark, unprocessed, dark subtracted or gain normalized + :param exposure: seconds + :param shutterDelay: milliseconds + """ + + arrSize = width * height + + # TODO: need to figure out what these should be + divideBy2 = 0 + settling = 0.0 + + if processing == 'dark': + longargs = [enum_gs['GS_GetDarkReference']] + else: + longargs = [enum_gs['GS_GetAcquiredImage']] + longargs.extend([ + arrSize, # pixels in the image + width, height + ]) + if processing == 'unprocessed': + longargs.append(0) + elif processing == 'dark subtracted': + longargs.append(1) + elif processing == 'gain normalized': + longargs.append(2) + + longargs.extend([binning, top, left, bottom, right, shutter]) + if processing != 'dark': + longargs.append(shutterDelay) + longargs.extend([divideBy2, corrections]) + dblargs = [exposure, settling] + + message_send = Message(longargs=longargs, dblargs=dblargs) + message_recv = Message(longargs=(0, 0, 0, 0, 0)) + + # attempt to solve UCLA problem by reconnecting + # if self.save_frames: + # self.reconnect() + + self.ExchangeMessages(message_send, message_recv) + + longargs = message_recv.array['longargs'] + if longargs[0] < 0: + return 1 + arrSize = longargs[1] + width = longargs[2] + height = longargs[3] + numChunks = longargs[4] + bytesPerPixel = 2 + numBytes = arrSize * bytesPerPixel + chunkSize = (numBytes + numChunks - 1) / numChunks + imArray = np.zeros((height, width), np.ushort) + received = 0 + remain = numBytes + for chunk in range(numChunks): + # send chunk handshake for all but the first chunk + if chunk: + message_send = Message(longargs=(enum_gs['GS_ChunkHandshake'],)) + self.ExchangeMessages(message_send) + thisChunkSize = min(remain, chunkSize) + chunkReceived = 0 + chunkRemain = thisChunkSize + while chunkRemain: + new_recv = self.recv_data(chunkRemain) + len_recv = len(new_recv) + imArray.data[received: received + len_recv] = new_recv + chunkReceived += len_recv + chunkRemain -= len_recv + remain -= len_recv + received += len_recv + return imArray + + def UpdateK2HardwareDarkReference(self, camera): + function_name = 'K2_updateHardwareDarkReference' + return self.ExecuteSendCameraObjectionFunction(function_name, camera) + + def FreeK2GainReference(self, value): + self.SetFunction('GS_FreeK2GainReference', slongargs=(value,)) + + def PrepareDarkReference(self, camera): + function_name = 'CM_PrepareDarkReference' + return self.ExecuteSendCameraObjectionFunction(function_name, camera) + +# ---------- Energy filter functions ------------------------------------------ + + def GetEnergyFilter(self): + if 'GetEnergyFilter' not in self.filter_functions.keys(): + return -1.0 + func = self.filter_functions['GetEnergyFilter'] + script = f'if ( {func}() ) {{ Exit(1.0); }} else {{ Exit(-1.0); }}' + return self.ExecuteGetDoubleScript(script) + + def SetEnergyFilter(self, value): + if 'SetEnergyFilter' not in self.filter_functions.keys(): + return -1.0 + if value: + i = 1 + else: + i = 0 + func = self.filter_functions['SetEnergyFilter'] + wait = self.wait_for_filter + script = f'{func}({i}); {wait}' + return self.ExecuteSendScript(script) + + def GetEnergyFilterWidth(self): + if 'GetEnergyFilterWidth' not in self.filter_functions.keys(): + return -1.0 + func = self.filter_functions['GetEnergyFilterWidth'] + script = f'Exit({func}())' + return self.ExecuteGetDoubleScript(script) + + def SetEnergyFilterWidth(self, value): + if 'SetEnergyFilterWidth' not in self.filter_functions.keys(): + return -1.0 + func = self.filter_functions['SetEnergyFilterWidth'] + script = f'if ( {func}({value:f}) ) {{ Exit(1.0); }} else {{ Exit(-1.0); }}' + return self.ExecuteSendScript(script) + + def GetEnergyFilterOffset(self): + if 'GetEnergyFilterOffset' not in self.filter_functions.keys(): + return 0.0 + func = self.filter_functions['GetEnergyFilterOffset'] + script = f'Exit({func}())' + return self.ExecuteGetDoubleScript(script) + + def SetEnergyFilterOffset(self, value): + if 'SetEnergyFilterOffset' not in self.filter_functions.keys(): + return -1.0 + func = self.filter_functions['SetEnergyFilterOffset'] + script = f'if ( {func}({value:f}) ) {{ Exit(1.0); }} else {{ Exit(-1.0); }}' + return self.ExecuteSendScript(script) + + def AlignEnergyFilterZeroLossPeak(self): + func = self.filter_functions['AlignEnergyFilterZeroLossPeak'] + wait = self.wait_for_filter + script = f' if ( {func}() ) {{ {wait} Exit(1.0); }} else {{ Exit(-1.0); }}' + return self.ExecuteGetDoubleScript(script) From a8558164603f7bbdb3d6c0f2b54ff7c7853afda6 Mon Sep 17 00:00:00 2001 From: Grigory Sharov Date: Wed, 19 Mar 2025 15:20:44 +0000 Subject: [PATCH 02/11] first prototype working --- pytemscript/plugins/gatan_sem_socket.py | 852 ++++++++++++------------ 1 file changed, 443 insertions(+), 409 deletions(-) diff --git a/pytemscript/plugins/gatan_sem_socket.py b/pytemscript/plugins/gatan_sem_socket.py index b3f9477..3a37afc 100644 --- a/pytemscript/plugins/gatan_sem_socket.py +++ b/pytemscript/plugins/gatan_sem_socket.py @@ -1,52 +1,33 @@ -""" -The code below is a modified version of: -https://github.com/instamatic-dev/instamatic/blob/main/src/instamatic/camera/gatansocket3.py +# The Leginon software is Copyright 2003 +# The Scripps Research Institute, La Jolla, CA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# The code below is a modified version of: https://github.com/leginon-org/leginon/blob/myami-python3/pyscope/gatansocket.py +# This plugin needs the SERIALEMCCD plugin to be installed in DigitalMicrograph +# The instructions can be found at: https://bio3d.colorado.edu/SerialEM/hlp/html/setting_up_serialem.htm +# +# On the computer running DM, define environment variables: +# SERIALEMCCD_PORT=48890 +# [optional] SERIALEMCCD_DEBUG=1 -BSD 3-Clause License - -Copyright (c) 2021, Stef Smeets -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The script adapted from [Leginon](https://emg.nysbc.org//projects/leginon/wiki/Leginon_Homepage). Leginon is licenced under the Apache License, Version 2.0. The code (`gatansocket3.py`) was converted from Python2.7 to Python3.6+ from [here](https://github.com/leginon-org/leginon/blob/myami-python3/pyscope/gatansocket.py). - -It needs the SERIALEMCCD plugin to be installed in DigitalMicrograph. The instructions can be found [here](https://bio3d.colorado.edu/SerialEM/hlp/html/setting_up_serialem.htm). - -- On the computer running DM, define an environment variable SERIALEMCCD_PORT=48890 -- [Optional] Set an environment variable SERIALEMCCD_DEBUG=1 or 2 - -""" - -import os -import socket import logging +import functools +import socket +from typing import Tuple import numpy as np -# enum function codes as in SocketPathway.cpp +# enum function codes as in https://github.com/mastcu/SerialEMCCD/blob/master/SocketPathway.cpp # need to match exactly both in number and order enum_gs = [ 'GS_ExecuteScript', @@ -72,7 +53,7 @@ 'GS_StopDSAcquisition', 'GS_CheckReferenceTime', 'GS_SetK2Parameters', - 'GS_ChunkHandshake', + 'GS_ChunkHandshake', # deleted in new plugin? 'GS_SetupFileSaving', 'GS_GetFileSaveResult', 'GS_SetupFileSaving2', @@ -94,9 +75,9 @@ 'GS_GetTiltSumProperties', ] # lookup table of function name to function code, starting with 1 -enum_gs = {x: y for (y, x) in enumerate(enum_gs, 1)} +enum_gs = {item: index for index, item in enumerate(enum_gs, 1)} -# C "long" -> numpy "int_" +## C "long" -> numpy "int32" ARGS_BUFFER_SIZE = 1024 MAX_LONG_ARGS = 16 MAX_DBL_ARGS = 8 @@ -105,75 +86,60 @@ class Message: - """Information packet to send and receive on the socket. - - Initialize with the sequences of args (longs, bools, doubles) and - optional long array. - """ + """ Information packet to send and receive on the socket. """ + def __init__(self, + longargs=(), boolargs=(), dblargs=(), + longarray=np.array([], dtype=np.int32)): + """ Initialize with the sequences of args (longs, bools, doubles) and optional long array. """ - def __init__(self, longargs=[], boolargs=[], dblargs=[], longarray=[]): - # Strings are packaged as long array using np.frombuffer(buffer,np.int_) - # and can be converted back with longarray.tostring() - # add final longarg with size of the longarray - if longarray: - longargs = list(longargs) - longargs.append(len(longarray)) + if len(longarray): + longargs = (*longargs, len(longarray)) self.dtype = [ - ('size', np.intc), - ('longargs', np.int_, (len(longargs),)), + ('size', np.intc, (1,)), + ('longargs', np.int32, (len(longargs),)), ('boolargs', np.int32, (len(boolargs),)), ('dblargs', np.double, (len(dblargs),)), - ('longarray', np.int_, (len(longarray),)), + ('longarray', np.int32, (len(longarray),)), ] - self.array = np.zeros((), dtype=self.dtype) - self.array['size'] = self.array.data.itemsize - self.array['longargs'] = longargs - self.array['boolargs'] = boolargs - self.array['dblargs'] = dblargs + self.array = np.empty((), dtype=self.dtype) + self.array['size'] = np.array([self.array.nbytes], dtype=np.intc) + self.array['longargs'] = np.array(longargs, dtype=np.int32) + self.array['boolargs'] = np.array(boolargs, dtype=np.int32) + self.array['dblargs'] = np.array(dblargs, dtype=np.double) self.array['longarray'] = longarray - # create numpy arrays for the args and array - # self.longargs = np.asarray(longargs, dtype=np.int_) - # self.dblargs = np.asarray(dblargs, dtype=np.double) - # self.boolargs = np.asarray(boolargs, dtype=np.int32) - # self.longarray = np.asarray(longarray, dtype=np.int_) - - def pack(self): - """Serialize the data.""" - data_size = self.array.data.itemsize - if self.array.data.itemsize > ARGS_BUFFER_SIZE: - raise RuntimeError(f'Message packet size {data_size} is larger than maximum {ARGS_BUFFER_SIZE}') + def pack(self) -> memoryview: + """ Serialize the data. """ + packed = self.array.nbytes + if packed > ARGS_BUFFER_SIZE: + raise RuntimeError('Message packet size %d is larger than maximum %d' % (packed, ARGS_BUFFER_SIZE)) return self.array.data - def unpack(self, buf): - """unpack buffer into our data structure.""" + def unpack(self, buf: bytes) -> None: + """ Unpack buffer into our data structure. """ self.array = np.frombuffer(buf, dtype=self.dtype)[0] def logwrap(func): - """Decorator for socket send and recv calls, so they can make log.""" - def newfunc(*args, **kwargs): - logging.debug(f'{func}\t{args}\t{kwargs}') + """ Decorator to log socket send and recv calls. """ + @functools.wraps(func) + def wrapper(*args, **kwargs): + logging.debug('%s\t%r\t%r', func.__name__, args, kwargs) try: - result = func(*args, **kwargs) - except Exception as ex: - logging.debug(f'EXCEPTION: {ex}') + return func(*args, **kwargs) + except Exception as e: + logging.error('EXCEPTION: %s', e) raise - return result - return newfunc + + return wrapper class GatanSocket: - def __init__(self, host, port): + def __init__(self, host="127.0.0.1", port=48890): + self.sock = None self.host = host - self.port = os.environ.get('SERIALEMCCD_PORT', port) - self.debug = os.environ.get('SERIALEMCCD_DEBUG', 0) - if self.debug: - logging.debug('GatanServerIP =', self.host) - logging.debug('SERIALEMCCD_PORT = GatanServerPort =', self.port) - logging.debug('SERIALEMCCD_DEBUG =', self.debug) - + self.port = port self.save_frames = False self.num_grab_sum = 0 self.connect() @@ -192,57 +158,53 @@ def __init__(self, host, port): ('IFGetSlitIn', 'GetEnergyFilter'), ('IFSetSlitIn', 'SetEnergyFilter'), ('IFGetEnergyLoss', 'GetEnergyFilterOffset'), - ('IFSetEnergyLoss', 'SetEnergyFilterOffset'), + ('IFSetEnergyOffset', 'SetEnergyFilterOffset'), # wjr this was IFSetEnergyLoss + ('IFGetMaximumSlitWidth', 'GetEnergyFilterWidthMax'), ('IFGetSlitWidth', 'GetEnergyFilterWidth'), ('IFSetSlitWidth', 'SetEnergyFilterWidth'), ('GT_CenterZLP', 'AlignEnergyFilterZeroLossPeak'), ] self.filter_functions = {} for name, method_name in self.script_functions: - hasScriptFunction = self.hasScriptFunction(name) if self.hasScriptFunction(name): self.filter_functions[method_name] = name - if self.debug: - logging.debug(name, method_name, hasScriptFunction) if ('SetEnergyFilter' in self.filter_functions.keys() and self.filter_functions['SetEnergyFilter'] == 'IFSetSlitIn'): self.wait_for_filter = 'IFWaitForFilter();' else: self.wait_for_filter = '' - def hasScriptFunction(self, name): - script = f'if ( DoesFunctionExist("{name}") ) {{ Exit(1.0); }} else {{ Exit(-1.0); }}' + def hasScriptFunction(self, name: str) -> bool: + script = 'if ( DoesFunctionExist("%s") ) { Exit(1.0); } else { Exit(-1.0); }' % name result = self.ExecuteGetDoubleScript(script) return result > 0.0 - def connect(self): + def connect(self) -> None: # recommended by Gatan to use localhost IP to avoid using tcp - self.sock = socket.create_connection(('127.0.0.1', self.port)) + self.sock = socket.create_connection((self.host, self.port)) - def disconnect(self): + def disconnect(self) -> None: self.sock.shutdown(socket.SHUT_RDWR) self.sock.close() - def reconnect(self): + def reconnect(self) -> None: self.disconnect() self.connect() @logwrap - def send_data(self, data): + def send_data(self, data: memoryview): return self.sock.sendall(data) @logwrap - def recv_data(self, n): + def recv_data(self, n: int) -> bytes: return self.sock.recv(n) def ExchangeMessages(self, message_send, message_recv=None): self.send_data(message_send.pack()) - if message_recv is None: return recv_buffer = message_recv.pack() - recv_len = recv_buffer.itemsize - + recv_len = recv_buffer.nbytes total_recv = 0 parts = [] while total_recv < recv_len: @@ -252,228 +214,89 @@ def ExchangeMessages(self, message_send, message_recv=None): total_recv += len(new_recv) buf = b''.join(parts) message_recv.unpack(buf) - # log the error code from received message + ## log the error code from received message sendargs = message_send.array['longargs'] recvargs = message_recv.array['longargs'] - logging.debug(f'Func: {sendargs[0]}, Code: {recvargs[0]}') + logging.debug('Func: %d, Code: %d', sendargs[0], recvargs[0]) - def GetFunction(self, funcName, rlongargs=[], rboolargs=[], rdblargs=[]): - """ Common function that only receives data. """ + def GetLong(self, funcName): + """ Common class of function that gets a single long """ funcCode = enum_gs[funcName] message_send = Message(longargs=(funcCode,)) - message_recv = Message(rlongargs, rboolargs, rdblargs) - self.ExchangeMessages(message_send, message_recv) - return message_recv - - def SetFunction(self, funcName, slongargs=[], sboolargs=[], sdblargs=[]): - """ Common function that only sends data. """ - funcCode = enum_gs[funcName] - message_send = Message(longargs=(funcCode, slongargs, sboolargs, sdblargs)) - message_recv = Message(longargs=(0,)) - self.ExchangeMessages(message_send, message_recv) - - def ExecuteSendCameraObjectionFunction(self, function_name, camera_id=0): - # first longargs is error code. Error if > 0 - return self.ExecuteGetLongCameraObjectFunction(function_name, camera_id) - - def ExecuteGetLongCameraObjectFunction(self, function_name, camera_id=0): - """Execute DM script function that requires camera object as input and - output one long integer.""" - recv_longargs_init = (0,) - result = self.ExecuteCameraObjectFunction(function_name, camera_id, - recv_longargs_init=recv_longargs_init) - if result is False: - return 1 - return result.array['longargs'][0] - - def ExecuteGetDoubleCameraObjectFunction(self, function_name, camera_id=0): - """Execute DM script function that requires camera object as input and - output double floating point number.""" - recv_dblargs_init = (0,) - result = self.ExecuteCameraObjectFunction(function_name, camera_id, - recv_dblargs_init=recv_dblargs_init) - if result is False: - return -999.0 - return result.array['dblargs'][0] - - def ExecuteCameraObjectFunction(self, function_name, camera_id=0, recv_longargs_init=(0,), - recv_dblargs_init=(0.0,), recv_longarray_init=[]): - """Execute DM script function that requires camera object as input.""" - if not self.hasScriptFunction(function_name): - # unsuccessful - return False - fullcommand = (f'Object manager = CM_GetCameraManager();\n' - f'Object cameraList = CM_GetCameras(manager);\n' - f'Object camera = ObjectAt(cameraList,{camera_id});\n' - f'{function_name}(camera);\n') - result = self.ExecuteScript(fullcommand, camera_id, recv_longargs_init, - recv_dblargs_init, recv_longarray_init) - return result - - def ExecuteSendScript(self, command_line, select_camera=0): - recv_longargs_init = (0,) - result = self.ExecuteScript(command_line, select_camera, recv_longargs_init) - # first longargs is error code. Error if > 0 - return result.array['longargs'][0] - - def ExecuteGetLongScript(self, command_line, select_camera=0): - """Execute DM script and return the result as integer.""" - # SerialEMCCD DM TemplatePlugIn::ExecuteScript retval is a double - return int(self.ExecuteGetDoubleScript(command_line, select_camera)) - - def ExecuteGetDoubleScript(self, command_line, select_camera=0): - """Execute DM script that gets one double float number.""" - recv_dblargs_init = (0.0,) - result = self.ExecuteScript(command_line, select_camera, recv_dblargs_init=recv_dblargs_init) - return result.array['dblargs'][0] - - def ExecuteScript(self, command_line, select_camera=0, recv_longargs_init=(0,), - recv_dblargs_init=(0.0,), recv_longarray_init=[]): - funcCode = enum_gs['GS_ExecuteScript'] - cmd_str = command_line + '\0' - extra = len(cmd_str) % 4 - if extra: - npad = 4 - extra - cmd_str = cmd_str + (npad) * '\0' - # send the command string as 1D longarray - longarray = np.frombuffer(cmd_str.encode(), dtype=np.int_) - # logging.debug(longaray) - message_send = Message(longargs=(funcCode,), boolargs=(select_camera,), longarray=longarray) - message_recv = Message(longargs=recv_longargs_init, dblargs=recv_dblargs_init, - longarray=recv_longarray_init) + # First recieved message longargs is error code + message_recv = Message(longargs=(0, 0)) self.ExchangeMessages(message_send, message_recv) - return message_recv - - def RunScript(self, fn: str, background: bool = False): - """Run a DM script. - - fn: str - Path to the script to run - background: bool - Prepend `// $BACKGROUND$` to run the script in the background - and make it non-blocking. - """ - - bkg = r'// $BACKGROUND$\n\n' - - with open(fn, 'r') as f: - cmd_str = ''.join(f.readlines()) - - if background: - cmd_str = bkg + cmd_str - - return self.ExecuteScript(cmd_str) - - -class SocketFuncs(GatanSocket): - def __init__(self, host='127.0.0.1', port=48890): - super().__init__(host, port) - -# ---------- DM functions ----------------------------------------------------- - - def GetDMVersion(self): - message_recv = self.GetFunction('GS_GetDMVersion', - rlongargs=(0, 0)) result = message_recv.array['longargs'][1] return result - def GetDMVersionAndBuild(self): - message_recv = self.GetFunction('GS_GetDMVersionAndBuild', - rlongargs=(0, 0, 0)) - result = message_recv.array['longargs'] - return result[0], result[1] - - def GetDMCapabilities(self): - message_recv = self.GetFunction('GS_GetDMCapabilities', - rlongargs=(0,), - rboolargs=(0, 0, 0)) - result = message_recv.array['boolargs'] - # canSelectShutter, canSetSettling, openShutterWorks - return list(map(bool, result)) - - def GetPluginVersion(self): - message_recv = self.GetFunction('GS_GetPluginVersion', - rlongargs=(0, 0)) - result = message_recv.array['longargs'][1] - return result - - def GetLastError(self): - message_recv = self.GetFunction('GS_GetLastError', - rlongargs=(0, 0)) + def SendLongGetLong(self, funcName, longarg): + """ Common class of function with one long arg that returns a single long """ + funcCode = enum_gs[funcName] + message_send = Message(longargs=(funcCode, longarg)) + # First recieved message longargs is error code + message_recv = Message(longargs=(0, 0)) + self.ExchangeMessages(message_send, message_recv) result = message_recv.array['longargs'][1] return result - def SetDebugMode(self, mode): - self.SetFunction('GS_SetDebugMode', slongargs=(mode,)) - -# ---------- Camera functions ------------------------------------------------- + def GetDMVersion(self): + return self.GetLong('GS_GetDMVersion') def GetNumberOfCameras(self): - message_recv = self.GetFunction('GS_GetNumberOfCameras', - rlongargs=(0, 0)) - result = message_recv.array['longargs'][1] - return result + return self.GetLong('GS_GetNumberOfCameras') - def SetCurrentCamera(self, camera): - self.SetFunction('GS_SetCurrentCamera', slongargs=(camera,)) - - def SelectCamera(self, camera): - self.SetFunction('GS_SelectCamera', - slongargs=(camera,)) + def GetPluginVersion(self): + return self.GetLong('GS_GetPluginVersion') - def IsCameraInserted(self, camera): + def IsCameraInserted(self, cameraid: int): funcCode = enum_gs['GS_IsCameraInserted'] - message_send = Message(longargs=(funcCode, camera)) + message_send = Message(longargs=(funcCode, cameraid)) message_recv = Message(longargs=(0,), boolargs=(0,)) self.ExchangeMessages(message_send, message_recv) result = bool(message_recv.array['boolargs'][0]) return result - def InsertCamera(self, camera, state): - self.SetFunction('GS_InsertCamera', - slongargs=(camera,), - sboolargs=(state,)) + def InsertCamera(self, cameraid: int, state): + funcCode = enum_gs['GS_InsertCamera'] + message_send = Message(longargs=(funcCode, cameraid), boolargs=(state,)) + message_recv = Message(longargs=(0,)) + self.ExchangeMessages(message_send, message_recv) + + def SetReadMode(self, mode, scaling=1.0): + funcCode = enum_gs['GS_SetReadMode'] + message_send = Message(longargs=(funcCode, mode), dblargs=(scaling,)) + message_recv = Message(longargs=(0,)) + self.ExchangeMessages(message_send, message_recv) - def SetReadMode(self, mode=-1, scaling=1.0): - """ - Set the read mode and the scaling factor - For K2, pass 0, 1, or 2 for linear, counting, super-res - For K3, pass 3 or 4 for linear or super-res: this is THE signal that it is a K3 - For camera not needing read mode, pass -1 - For OneView, pass -3 for regular imaging or -2 for diffraction - For K3, the offset to be subtracted for linear mode must be supplied with the scaling. - The offset is supposed to be 8192 per frame - The offset per ms is thus nominally (8192 per frame) / (1.502 frames per ms) - pass scaling = trueScaling + 10 * nearestInt(offsetPerMs) - """ - self.SetFunction('GS_SetReadMode', - slongargs=(mode,), - sdblargs=(scaling,)) - - def SetShutterNormallyClosed(self, camera, shutter): - self.SetFunction('GS_SetShutterNormallyClosed', - slongargs=(camera, shutter,)) - - def GetLastDoseRate(self): - message_recv = self.GetFunction('GS_GetLastDoseRate', - rlongargs=(0,), - rdblargs=(0,)) - result = float(message_recv.array['dblargs']) - return result + def SetShutterNormallyClosed(self, cameraid: int, shutter): + funcCode = enum_gs['GS_SetShutterNormallyClosed'] + message_send = Message(longargs=(funcCode, cameraid, shutter)) + message_recv = Message(longargs=(0,)) + self.ExchangeMessages(message_send, message_recv) @logwrap - def SetK2Parameters(self, readMode, scaling, hardwareProc, doseFrac, - frameTime, alignFrames, saveFrames, filt='', useCds=False): - funcCode = enum_gs['GS_SetK2Parameters'] - + def SetK2Parameters(self, + readMode, + scaling, + hardwareProc, + doseFrac, + frameTime, + alignFrames, + saveFrames, + filt='', + useCds=False): + funcCode = enum_gs['GS_SetK2Parameters2'] # rotation and flip for non-frame saving image. It is the same definition # as in SetFileSaving2 - # if set to 0, it takes what GMS has.self.save_frames = saveFrames + # if set to 0, it takes what GMS has. rotationFlip = 0 + self.save_frames = saveFrames # flags flags = 0 flags += int(useCds) * 2 ** 6 + # settings of unused flags + # anti_alias reducedSizes = 0 fullSizes = 0 @@ -483,14 +306,14 @@ def SetK2Parameters(self, readMode, scaling, hardwareProc, doseFrac, if extra: npad = 4 - extra filt_str = filt_str + npad * '\0' - longarray = np.frombuffer(filt_str.encode(), dtype=np.int_) + longarray = np.frombuffer(bytes(filt_str, 'utf-8'), dtype=np.int32) longs = [ funcCode, readMode, hardwareProc, rotationFlip, - flags + flags, ] bools = [ doseFrac, @@ -506,21 +329,26 @@ def SetK2Parameters(self, readMode, scaling, hardwareProc, doseFrac, 0.0, # dummy4 ] - message_send = Message(longargs=longs, boolargs=bools, - dblargs=doubles, longarray=longarray) + message_send = Message(longargs=longs, boolargs=bools, dblargs=doubles, longarray=longarray) message_recv = Message(longargs=(0,)) # just return code self.ExchangeMessages(message_send, message_recv) - def setNumGrabSum(self, earlyReturnFrameCount, earlyReturnRamGrabs): + def setNumGrabSum(self, earlyReturnFrameCount: int, earlyReturnRamGrabs: int): # pack RamGrabs and earlyReturnFrameCount in one double - self.num_grab_sum = (2**16) * earlyReturnRamGrabs + earlyReturnFrameCount + self.num_grab_sum = 2**16 * earlyReturnRamGrabs + earlyReturnFrameCount def getNumGrabSum(self): return self.num_grab_sum @logwrap - def SetupFileSaving(self, rotationFlip, dirname, rootname, filePerImage, - doEarlyReturn, earlyReturnFrameCount=0, earlyReturnRamGrabs=0, + def SetupFileSaving(self, + rotationFlip, + dirname, + rootname, + filePerImage, + doEarlyReturn, + earlyReturnFrameCount=0, + earlyReturnRamGrabs=0, lzwtiff=False): pixelSize = 1.0 self.setNumGrabSum(earlyReturnFrameCount, earlyReturnRamGrabs) @@ -529,55 +357,137 @@ def SetupFileSaving(self, rotationFlip, dirname, rootname, filePerImage, flag = 128 * int(doEarlyReturn) + 8 * int(lzwtiff) numGrabSum = self.getNumGrabSum() # set values to pass - longs = [enum_gs['GS_SetupFileSaving2'], rotationFlip, flag] - dbls = [pixelSize, numGrabSum, 0., 0., 0.] + longs = [enum_gs['GS_SetupFileSaving2'], rotationFlip, flag, ] + dbls = [pixelSize, numGrabSum, 0., 0., 0., ] else: - longs = [enum_gs['GS_SetupFileSaving'], rotationFlip] - dbls = [pixelSize] - bools = [filePerImage] + longs = [enum_gs['GS_SetupFileSaving'], rotationFlip, ] + dbls = [pixelSize, ] + bools = [filePerImage, ] names_str = dirname + '\0' + rootname + '\0' extra = len(names_str) % 4 if extra: npad = 4 - extra names_str = names_str + npad * '\0' - longarray = np.frombuffer(names_str.encode(), dtype=np.int_) - message_send = Message(longargs=longs, boolargs=bools, - dblargs=dbls, longarray=longarray) + longarray = np.frombuffer(bytes(names_str, 'utf-8'), dtype=np.int32) + message_send = Message(longargs=longs, boolargs=bools, dblargs=dbls, longarray=longarray) message_recv = Message(longargs=(0, 0)) self.ExchangeMessages(message_send, message_recv) - def StopDSAcquisition(self): - message_recv = self.GetFunction('GS_StopDSAcquisition', - rlongargs=(0, )) + def GetFileSaveResult(self): + #longs = [enum_gs['GS_GetFileSaveResult'], rotationFlip] + message_send = Message(longargs=(enum_gs['GS_GetFileSaveResult'],))#, boolargs=bools, dblargs=dbls, longarray=longarray) + message_recv = Message(longargs=(0, 0, 0)) + self.ExchangeMessages(message_send, message_recv) + args = message_recv.array['longargs'] + numsaved = args[1] + error = args[2] + return numsaved, error + + def SelectCamera(self, cameraid: int): + funcCode = enum_gs['GS_SelectCamera'] + message_send = Message(longargs=(funcCode, cameraid)) + message_recv = Message(longargs=(0,)) + self.ExchangeMessages(message_send, message_recv) + + def UpdateK2HardwareDarkReference(self, cameraid: int): + function_name = 'K2_updateHardwareDarkReference' + return self.ExecuteSendCameraObjectionFunction(function_name, cameraid) - def StopContinuousCamera(self): - message_recv = self.GetFunction('GS_StopContinuousCamera', - rlongargs=(0, )) + def PrepareDarkReference(self, cameraid: int): + function_name = 'CM_PrepareDarkReference' + return self.ExecuteSendCameraObjectionFunction(function_name, cameraid) - def GetFileSaveResult(self): - message_recv = self.GetFunction('GS_GetFileSaveResult', - rlongargs=(0, 0, 0)) - result = message_recv.array['longargs'] - # result = numSaved, error - return result[1] + def GetEnergyFilter(self): + if 'GetEnergyFilter' not in list(self.filter_functions.keys()): + return -1.0 + script = 'if ( %s() ) { Exit(1.0); } else { Exit(-1.0); }' % (self.filter_functions['GetEnergyFilter'],) + return self.ExecuteGetDoubleScript(script) - def SetNoDMSettling(self, value): - self.SetFunction('GS_SetNoDMSettling', - slongargs=(value,)) + def SetEnergyFilter(self, value): + if 'SetEnergyFilter' not in list(self.filter_functions.keys()): + return -1.0 + if value: + i = 1 + else: + i = 0 + script = '%s(%d); %s' % (self.filter_functions['SetEnergyFilter'], i, self.wait_for_filter) + return self.ExecuteSendScript(script) - def WaitUntilReady(self, value): - self.SetFunction('GS_WaitUntilReady', - slongargs=(value,)) + def GetEnergyFilterWidthMax(self): + if 'GetEnergyFilterWidthMax' not in list(self.filter_functions.keys()): + return -1.0 + script = 'Exit(%s())' % (self.filter_functions['GetEnergyFilterWidthMax'],) + return self.ExecuteGetDoubleScript(script) - @logwrap - def GetImage(self, processing, height, width, binning, top, - left, bottom, right, exposure, corrections, - shutter=0, shutterDelay=0.): + def GetEnergyFilterWidth(self): + if 'GetEnergyFilterWidth' not in list(self.filter_functions.keys()): + return -1.0 + script = 'Exit(%s())' % (self.filter_functions['GetEnergyFilterWidth'],) + return self.ExecuteGetDoubleScript(script) + + def SetEnergyFilterWidth(self, value): + if 'SetEnergyFilterWidth' not in list(self.filter_functions.keys()): + return -1.0 + script = 'if ( %s(%f) ) { Exit(1.0); } else { Exit(-1.0); }' % ( + self.filter_functions['SetEnergyFilterWidth'], value) + return self.ExecuteSendScript(script) + + def GetEnergyFilterOffset(self): + if 'GetEnergyFilterOffset' not in list(self.filter_functions.keys()): + return 0.0 + script = 'Exit(%s())' % (self.filter_functions['GetEnergyFilterOffset'],) + return self.ExecuteGetDoubleScript(script) + + def SetEnergyFilterOffset(self, value): """ - :param processing: dark, unprocessed, dark subtracted or gain normalized - :param exposure: seconds - :param shutterDelay: milliseconds + wjr changing this to use the Gatan function IFSetEnergyOffset, which needs a technique and a value + GMS 3.32,function apparently added in GMS 3.2. Later versions will need to be checked + technique 0: instrument (not available) + technique 1: prism offset (confusing because -10 is 10) + technique 2: HT offset (has no effect on BioQuantum, but HT offset in DM will change) + technique 3: drift tube ( this seems best since the value is consistant with direction) + technique 4: prism adjust (confusing because -10 is 10 and it does not count when checking the energy loss value) + note: the Gatan function being called is a void, so removed the boolean logic used for most other functions """ + technique = 3 # hard code to drift tube for now + if 'SetEnergyFilterOffset' not in list(self.filter_functions.keys()): + return -1.0 + script = '%s(%i,%f)' % (self.filter_functions['SetEnergyFilterOffset'], technique, value) + self.ExecuteSendScript(script) + # return 1 + # or better to + newvalue = self.GetEnergyFilterOffset() # ? but wastes time + if value == newvalue: + return 1 + else: + technique = 2 # reset the HT offfset to 0, sometimes this gets set + script = '%s(%i,%f)' % (self.filter_functions['SetEnergyFilterOffset'], technique, 0) + self.ExecuteSendScript(script) + newvalue = self.GetEnergyFilterOffset() + if value == newvalue: + return 1 + else: + return -1 + + def AlignEnergyFilterZeroLossPeak(self): + script = ' if ( %s() ) { %s Exit(1.0); } else { Exit(-1.0); }' % ( + self.filter_functions['AlignEnergyFilterZeroLossPeak'], self.wait_for_filter) + return self.ExecuteGetDoubleScript(script) + + @logwrap + def GetImage(self, + processing, + height, + width, + binning, + top, + left, + bottom, + right, + exposure, + corrections, + shutter=0, + shutterDelay=0.0): arrSize = width * height @@ -585,13 +495,14 @@ def GetImage(self, processing, height, width, binning, top, divideBy2 = 0 settling = 0.0 + # prepare args for message if processing == 'dark': longargs = [enum_gs['GS_GetDarkReference']] else: longargs = [enum_gs['GS_GetAcquiredImage']] longargs.extend([ arrSize, # pixels in the image - width, height + width, height, ]) if processing == 'unprocessed': longargs.append(0) @@ -599,12 +510,21 @@ def GetImage(self, processing, height, width, binning, top, longargs.append(1) elif processing == 'gain normalized': longargs.append(2) - - longargs.extend([binning, top, left, bottom, right, shutter]) + longargs.extend([ + binning, + top, left, bottom, right, + shutter, + ]) if processing != 'dark': longargs.append(shutterDelay) - longargs.extend([divideBy2, corrections]) - dblargs = [exposure, settling] + longargs.extend([ + divideBy2, + corrections, + ]) + dblargs = [ + exposure, + settling, + ] message_send = Message(longargs=longargs, dblargs=dblargs) message_recv = Message(longargs=(0, 0, 0, 0, 0)) @@ -615,20 +535,24 @@ def GetImage(self, processing, height, width, binning, top, self.ExchangeMessages(message_send, message_recv) - longargs = message_recv.array['longargs'] + longargs = message_recv.array['longargs'].tolist() + logging.debug('GetImage longargs %s', longargs) if longargs[0] < 0: return 1 arrSize = longargs[1] width = longargs[2] height = longargs[3] numChunks = longargs[4] - bytesPerPixel = 2 + bytesPerPixel = 2 # depends on the results formated =uint16 numBytes = arrSize * bytesPerPixel - chunkSize = (numBytes + numChunks - 1) / numChunks - imArray = np.zeros((height, width), np.ushort) + chunkSize = (numBytes + numChunks - 1) // numChunks + imArray = np.zeros((height * width,), np.uint16) received = 0 remain = numBytes + index = 0 + logging.debug('chunk size %d', chunkSize) for chunk in range(numChunks): + recv_bytes = b'' # send chunk handshake for all but the first chunk if chunk: message_send = Message(longargs=(enum_gs['GS_ChunkHandshake'],)) @@ -639,75 +563,185 @@ def GetImage(self, processing, height, width, binning, top, while chunkRemain: new_recv = self.recv_data(chunkRemain) len_recv = len(new_recv) - imArray.data[received: received + len_recv] = new_recv + recv_bytes += new_recv chunkReceived += len_recv chunkRemain -= len_recv remain -= len_recv received += len_recv + last_index = int(index) + logging.debug('chunk bytes length %d', len(recv_bytes)) + index = chunkSize * (chunk + 1) // bytesPerPixel + logging.debug('array index range to fill %d:%d', last_index, index) + imArray[last_index:index] = np.frombuffer(recv_bytes, dtype=np.uint16) + imArray = imArray.reshape((height, width)) return imArray - def UpdateK2HardwareDarkReference(self, camera): - function_name = 'K2_updateHardwareDarkReference' - return self.ExecuteSendCameraObjectionFunction(function_name, camera) - - def FreeK2GainReference(self, value): - self.SetFunction('GS_FreeK2GainReference', slongargs=(value,)) + def ExecuteSendCameraObjectionFunction(self, function_name, camera_id=0): + # first longargs is error code. Error if > 0 + return self.ExecuteGetLongCameraObjectFunction(function_name, camera_id) - def PrepareDarkReference(self, camera): - function_name = 'CM_PrepareDarkReference' - return self.ExecuteSendCameraObjectionFunction(function_name, camera) + def ExecuteGetLongCameraObjectFunction(self, function_name, camera_id=0): + """ Execute DM script function that requires camera object + as input and output one long integer. + """ + recv_longargs_init = (0,) + result = self.ExecuteCameraObjectFunction(function_name, camera_id, + recv_longargs_init=recv_longargs_init) + if result is False: + return 1 + return result.array['longargs'][0] -# ---------- Energy filter functions ------------------------------------------ + def ExecuteGetDoubleCameraObjectFunction(self, function_name, camera_id=0): + """ Execute DM script function that requires camera object + as input and output double floating point number. + """ + recv_dblargs_init = (0,) + result = self.ExecuteCameraObjectFunction(function_name, camera_id, + recv_dblargs_init=recv_dblargs_init) + if result is False: + return -999.0 + return result.array['dblargs'][0] - def GetEnergyFilter(self): - if 'GetEnergyFilter' not in self.filter_functions.keys(): - return -1.0 - func = self.filter_functions['GetEnergyFilter'] - script = f'if ( {func}() ) {{ Exit(1.0); }} else {{ Exit(-1.0); }}' - return self.ExecuteGetDoubleScript(script) + def ExecuteCameraObjectFunction(self, + function_name, + camera_id=0, + recv_longargs_init=(0,), + recv_dblargs_init=(0.0,)): + """ Execute DM script function that requires camera object as input. """ + if not self.hasScriptFunction(function_name): + return False + fullcommand = "Object manager = CM_GetCameraManager();\n Object cameraList = CM_GetCameras(manager);\n Object camera = ObjectAt(cameraList,%d);\n " % ( + camera_id) + fullcommand += "%s(camera);\n" % function_name + result = self.ExecuteScript(fullcommand, camera_id, recv_longargs_init, + recv_dblargs_init) + return result - def SetEnergyFilter(self, value): - if 'SetEnergyFilter' not in self.filter_functions.keys(): - return -1.0 - if value: - i = 1 - else: - i = 0 - func = self.filter_functions['SetEnergyFilter'] - wait = self.wait_for_filter - script = f'{func}({i}); {wait}' - return self.ExecuteSendScript(script) + def ExecuteSendScript(self, command_line, select_camera=0): + recv_longargs_init = (0,) + result = self.ExecuteScript(command_line, select_camera, recv_longargs_init) + # first longargs is error code. Error if > 0 + return result.array['longargs'][0] - def GetEnergyFilterWidth(self): - if 'GetEnergyFilterWidth' not in self.filter_functions.keys(): - return -1.0 - func = self.filter_functions['GetEnergyFilterWidth'] - script = f'Exit({func}())' - return self.ExecuteGetDoubleScript(script) + def ExecuteGetLongScript(self, command_line, select_camera=0): + """ Execute DM script and return the result as integer. """ + # SerialEMCCD DM TemplatePlugIn::ExecuteScript retval is a double + return int(self.ExecuteGetDoubleScript(command_line, select_camera)) - def SetEnergyFilterWidth(self, value): - if 'SetEnergyFilterWidth' not in self.filter_functions.keys(): - return -1.0 - func = self.filter_functions['SetEnergyFilterWidth'] - script = f'if ( {func}({value:f}) ) {{ Exit(1.0); }} else {{ Exit(-1.0); }}' - return self.ExecuteSendScript(script) + def ExecuteGetDoubleScript(self, command_line, select_camera=0): + """ Execute DM script that gets one double float number. """ + result = self.ExecuteScript(command_line, select_camera) + return result.array['dblargs'][0] - def GetEnergyFilterOffset(self): - if 'GetEnergyFilterOffset' not in self.filter_functions.keys(): - return 0.0 - func = self.filter_functions['GetEnergyFilterOffset'] - script = f'Exit({func}())' - return self.ExecuteGetDoubleScript(script) + def ExecuteScript(self, + command_line: str, + select_camera: int = 0, + recv_longargs_init: Tuple = (0,), + recv_dblargs_init: Tuple = (0.0,)): + funcCode = enum_gs['GS_ExecuteScript'] + cmd_str = command_line + '\0' + extra = len(cmd_str) % 4 + if extra: + npad = 4 - extra + cmd_str = cmd_str + npad * '\0' + # send the command string as 1D longarray + longarray = np.frombuffer(bytes(cmd_str, 'utf-8'), dtype=np.int32) + message_send = Message(longargs=(funcCode,), boolargs=(select_camera,), longarray=longarray) + message_recv = Message(longargs=recv_longargs_init, dblargs=recv_dblargs_init) + self.ExchangeMessages(message_send, message_recv) + return message_recv - def SetEnergyFilterOffset(self, value): - if 'SetEnergyFilterOffset' not in self.filter_functions.keys(): - return -1.0 - func = self.filter_functions['SetEnergyFilterOffset'] - script = f'if ( {func}({value:f}) ) {{ Exit(1.0); }} else {{ Exit(-1.0); }}' - return self.ExecuteSendScript(script) - def AlignEnergyFilterZeroLossPeak(self): - func = self.filter_functions['AlignEnergyFilterZeroLossPeak'] - wait = self.wait_for_filter - script = f' if ( {func}() ) {{ {wait} Exit(1.0); }} else {{ Exit(-1.0); }}' - return self.ExecuteGetDoubleScript(script) +if __name__ == '__main__': + g = GatanSocket(host="192.168.71.2") + print(g) + print('Version', g.GetDMVersion()) + print('GetNumberOfCameras', g.GetNumberOfCameras()) + print('GetPluginVersion', g.GetPluginVersion()) + if not g.IsCameraInserted(0): + print('InsertCamera') + g.InsertCamera(0, True) + print('IsCameraInserted', g.IsCameraInserted(0)) + + + k2params = { + 'readMode': 1, # {'non-K2 cameras':-1, 'linear': 0, 'counting': 1, 'super resolution': 2} + 'scaling': 1.0, + 'hardwareProc': 1, #{'none': 0, 'dark': 2, 'gain': 4, 'dark+gain': 6} + 'doseFrac': False, + 'frameTime': 0.25, + 'alignFrames': False, + 'saveFrames': False, + 'filt': 'None' + } + + def getDMVersion(g): + ''' + version: version_long, major.minor.sub + ''' + version_long = g.GetDMVersion() + if version_long < 40000: + major = 1 + minor = None + sub = None + if version_long >= 31100: + # 2.3.0 gives an odd number of 31100 + major = 2 + minor = 3 + sub = 0 + version_long = 40300 + elif version_long == 40000: + # minor version can be 0 or 1 in this case + # but likely 1 since we have not used this module until k2 is around + major = 2 + minor = 1 + sub = 0 + else: + major = version_long // 10000 - 2 + remainder = version_long - (major + 2) * 10000 + minor = remainder // 100 + sub = remainder % 100 + return (version_long, '%d.%d.%d' % (major, minor, sub)) + + def isDM332orUp(g): + version_id, version_string = getDMVersion(g) + if version_id and version_id >= 50302: + return True + return False + + + def getCorrectionFlags(g): + ''' + Binnary Correction flag sum in GMS. See Feature #8391. + GMS3.3.2 has pre-counting correction which is superior. + SerialEM always do this correction + but Leginon 3.4 and earlier does not. + David M. said SerialEM default is 49 for K2 and 1 for K3. + 49 means defect,bias, and quadrant (to be the same as Ultrascan). + I don't think the latter two needs applying in counting. + ''' + if isDM332orUp(g): + return 1 # defect correction only. + else: + # keep it zero to be back compatible. + return 0 + + acqparams = { + 'processing': 'unprocessed', # dark, dark subtracted, gain normalized + 'height': 2048, + 'width': 2048, + 'binning': 1, + 'top': 0, + 'left': 0, + 'bottom': 2048, + 'right': 2048, + 'exposure': 1.0, + 'corrections': getCorrectionFlags(g), + 'shutter': 0, + 'shutterDelay': 0.0, + } + + #g.SetK2Parameters(**k2params) + g.SetReadMode(-1) + image = g.GetImage(**acqparams) + print("Got image array:", image.dtype, image.shape) From d8d590d4f3fc453b4dfa9341ab49c0b8417c74a1 Mon Sep 17 00:00:00 2001 From: Grigory Sharov Date: Wed, 19 Mar 2025 16:21:08 +0000 Subject: [PATCH 03/11] add more typehints --- pytemscript/plugins/gatan_sem_socket.py | 239 ++++++++++++------------ 1 file changed, 119 insertions(+), 120 deletions(-) diff --git a/pytemscript/plugins/gatan_sem_socket.py b/pytemscript/plugins/gatan_sem_socket.py index 3a37afc..f0eca42 100644 --- a/pytemscript/plugins/gatan_sem_socket.py +++ b/pytemscript/plugins/gatan_sem_socket.py @@ -24,7 +24,7 @@ import logging import functools import socket -from typing import Tuple +from typing import Tuple, Optional import numpy as np # enum function codes as in https://github.com/mastcu/SerialEMCCD/blob/master/SocketPathway.cpp @@ -53,7 +53,7 @@ 'GS_StopDSAcquisition', 'GS_CheckReferenceTime', 'GS_SetK2Parameters', - 'GS_ChunkHandshake', # deleted in new plugin? + 'GS_ChunkHandshake', # deleted in the new plugin? 'GS_SetupFileSaving', 'GS_GetFileSaveResult', 'GS_SetupFileSaving2', @@ -136,7 +136,7 @@ def wrapper(*args, **kwargs): class GatanSocket: - def __init__(self, host="127.0.0.1", port=48890): + def __init__(self, host: str = "127.0.0.1", port: int = 48890): self.sock = None self.host = host self.port = port @@ -192,14 +192,15 @@ def reconnect(self) -> None: self.connect() @logwrap - def send_data(self, data: memoryview): - return self.sock.sendall(data) + def send_data(self, data: memoryview) -> None: + self.sock.sendall(data) @logwrap def recv_data(self, n: int) -> bytes: return self.sock.recv(n) - def ExchangeMessages(self, message_send, message_recv=None): + def ExchangeMessages(self, message_send: Message, + message_recv: Optional[Message] = None): self.send_data(message_send.pack()) if message_recv is None: return @@ -219,36 +220,36 @@ def ExchangeMessages(self, message_send, message_recv=None): recvargs = message_recv.array['longargs'] logging.debug('Func: %d, Code: %d', sendargs[0], recvargs[0]) - def GetLong(self, funcName): + def GetLong(self, funcName: str) -> int: """ Common class of function that gets a single long """ funcCode = enum_gs[funcName] message_send = Message(longargs=(funcCode,)) # First recieved message longargs is error code message_recv = Message(longargs=(0, 0)) self.ExchangeMessages(message_send, message_recv) - result = message_recv.array['longargs'][1] + result = int(message_recv.array['longargs'][1]) return result - def SendLongGetLong(self, funcName, longarg): + def SendLongGetLong(self, funcName: str, longarg) -> int: """ Common class of function with one long arg that returns a single long """ funcCode = enum_gs[funcName] message_send = Message(longargs=(funcCode, longarg)) # First recieved message longargs is error code message_recv = Message(longargs=(0, 0)) self.ExchangeMessages(message_send, message_recv) - result = message_recv.array['longargs'][1] + result = int(message_recv.array['longargs'][1]) return result - def GetDMVersion(self): + def GetDMVersion(self) -> int: return self.GetLong('GS_GetDMVersion') - def GetNumberOfCameras(self): + def GetNumberOfCameras(self) -> int: return self.GetLong('GS_GetNumberOfCameras') - def GetPluginVersion(self): + def GetPluginVersion(self) -> int: return self.GetLong('GS_GetPluginVersion') - def IsCameraInserted(self, cameraid: int): + def IsCameraInserted(self, cameraid: int) -> bool: funcCode = enum_gs['GS_IsCameraInserted'] message_send = Message(longargs=(funcCode, cameraid)) message_recv = Message(longargs=(0,), boolargs=(0,)) @@ -256,19 +257,19 @@ def IsCameraInserted(self, cameraid: int): result = bool(message_recv.array['boolargs'][0]) return result - def InsertCamera(self, cameraid: int, state): + def InsertCamera(self, cameraid: int, state: int): funcCode = enum_gs['GS_InsertCamera'] message_send = Message(longargs=(funcCode, cameraid), boolargs=(state,)) message_recv = Message(longargs=(0,)) self.ExchangeMessages(message_send, message_recv) - def SetReadMode(self, mode, scaling=1.0): + def SetReadMode(self, mode: int, scaling: float = 1.0): funcCode = enum_gs['GS_SetReadMode'] message_send = Message(longargs=(funcCode, mode), dblargs=(scaling,)) message_recv = Message(longargs=(0,)) self.ExchangeMessages(message_send, message_recv) - def SetShutterNormallyClosed(self, cameraid: int, shutter): + def SetShutterNormallyClosed(self, cameraid: int, shutter: int): funcCode = enum_gs['GS_SetShutterNormallyClosed'] message_send = Message(longargs=(funcCode, cameraid, shutter)) message_recv = Message(longargs=(0,)) @@ -276,23 +277,21 @@ def SetShutterNormallyClosed(self, cameraid: int, shutter): @logwrap def SetK2Parameters(self, - readMode, - scaling, - hardwareProc, - doseFrac, - frameTime, - alignFrames, - saveFrames, - filt='', - useCds=False): + readMode: int, + scaling: float = 1.0, + hardwareProc: int = 1, + doseFrac: bool = False, + frameTime: float = 0.25, + alignFrames: bool = False, + saveFrames: bool = False, + filt: str = '', + useCds: bool = False): funcCode = enum_gs['GS_SetK2Parameters2'] # rotation and flip for non-frame saving image. It is the same definition - # as in SetFileSaving2 - # if set to 0, it takes what GMS has. + # as in SetFileSaving2. If set to 0, it takes the value from GMS rotationFlip = 0 self.save_frames = saveFrames - # flags flags = 0 flags += int(useCds) * 2 ** 6 # settings of unused flags @@ -337,19 +336,19 @@ def setNumGrabSum(self, earlyReturnFrameCount: int, earlyReturnRamGrabs: int): # pack RamGrabs and earlyReturnFrameCount in one double self.num_grab_sum = 2**16 * earlyReturnRamGrabs + earlyReturnFrameCount - def getNumGrabSum(self): + def getNumGrabSum(self) -> int: return self.num_grab_sum @logwrap def SetupFileSaving(self, rotationFlip, - dirname, - rootname, - filePerImage, - doEarlyReturn, - earlyReturnFrameCount=0, - earlyReturnRamGrabs=0, - lzwtiff=False): + dirname: str, + rootname: str, + filePerImage: int, + doEarlyReturn: bool = False, + earlyReturnFrameCount: int = 0, + earlyReturnRamGrabs: int = 0, + lzwtiff: bool = False) -> None: pixelSize = 1.0 self.setNumGrabSum(earlyReturnFrameCount, earlyReturnRamGrabs) if self.save_frames and (doEarlyReturn or lzwtiff): @@ -373,7 +372,7 @@ def SetupFileSaving(self, message_recv = Message(longargs=(0, 0)) self.ExchangeMessages(message_send, message_recv) - def GetFileSaveResult(self): + def GetFileSaveResult(self) -> Tuple: #longs = [enum_gs['GS_GetFileSaveResult'], rotationFlip] message_send = Message(longargs=(enum_gs['GS_GetFileSaveResult'],))#, boolargs=bools, dblargs=dbls, longarray=longarray) message_recv = Message(longargs=(0, 0, 0)) @@ -389,23 +388,23 @@ def SelectCamera(self, cameraid: int): message_recv = Message(longargs=(0,)) self.ExchangeMessages(message_send, message_recv) - def UpdateK2HardwareDarkReference(self, cameraid: int): + def UpdateK2HardwareDarkReference(self, cameraid: int) -> int: function_name = 'K2_updateHardwareDarkReference' return self.ExecuteSendCameraObjectionFunction(function_name, cameraid) - def PrepareDarkReference(self, cameraid: int): + def PrepareDarkReference(self, cameraid: int) -> int: function_name = 'CM_PrepareDarkReference' return self.ExecuteSendCameraObjectionFunction(function_name, cameraid) - def GetEnergyFilter(self): - if 'GetEnergyFilter' not in list(self.filter_functions.keys()): + def GetEnergyFilter(self) -> float: + if 'GetEnergyFilter' not in self.filter_functions: return -1.0 script = 'if ( %s() ) { Exit(1.0); } else { Exit(-1.0); }' % (self.filter_functions['GetEnergyFilter'],) return self.ExecuteGetDoubleScript(script) - def SetEnergyFilter(self, value): - if 'SetEnergyFilter' not in list(self.filter_functions.keys()): - return -1.0 + def SetEnergyFilter(self, value: bool = False) -> int: + if 'SetEnergyFilter' not in self.filter_functions: + return -1 if value: i = 1 else: @@ -413,32 +412,32 @@ def SetEnergyFilter(self, value): script = '%s(%d); %s' % (self.filter_functions['SetEnergyFilter'], i, self.wait_for_filter) return self.ExecuteSendScript(script) - def GetEnergyFilterWidthMax(self): - if 'GetEnergyFilterWidthMax' not in list(self.filter_functions.keys()): + def GetEnergyFilterWidthMax(self) -> float: + if 'GetEnergyFilterWidthMax' not in self.filter_functions: return -1.0 script = 'Exit(%s())' % (self.filter_functions['GetEnergyFilterWidthMax'],) return self.ExecuteGetDoubleScript(script) - def GetEnergyFilterWidth(self): - if 'GetEnergyFilterWidth' not in list(self.filter_functions.keys()): + def GetEnergyFilterWidth(self) -> float: + if 'GetEnergyFilterWidth' not in self.filter_functions: return -1.0 script = 'Exit(%s())' % (self.filter_functions['GetEnergyFilterWidth'],) return self.ExecuteGetDoubleScript(script) - def SetEnergyFilterWidth(self, value): - if 'SetEnergyFilterWidth' not in list(self.filter_functions.keys()): - return -1.0 + def SetEnergyFilterWidth(self, value: float) -> int: + if 'SetEnergyFilterWidth' not in self.filter_functions: + return -1 script = 'if ( %s(%f) ) { Exit(1.0); } else { Exit(-1.0); }' % ( self.filter_functions['SetEnergyFilterWidth'], value) return self.ExecuteSendScript(script) - def GetEnergyFilterOffset(self): - if 'GetEnergyFilterOffset' not in list(self.filter_functions.keys()): + def GetEnergyFilterOffset(self) -> float: + if 'GetEnergyFilterOffset' not in self.filter_functions: return 0.0 script = 'Exit(%s())' % (self.filter_functions['GetEnergyFilterOffset'],) return self.ExecuteGetDoubleScript(script) - def SetEnergyFilterOffset(self, value): + def SetEnergyFilterOffset(self, value: float) -> float: """ wjr changing this to use the Gatan function IFSetEnergyOffset, which needs a technique and a value GMS 3.32,function apparently added in GMS 3.2. Later versions will need to be checked @@ -450,7 +449,7 @@ def SetEnergyFilterOffset(self, value): note: the Gatan function being called is a void, so removed the boolean logic used for most other functions """ technique = 3 # hard code to drift tube for now - if 'SetEnergyFilterOffset' not in list(self.filter_functions.keys()): + if 'SetEnergyFilterOffset' not in self.filter_functions: return -1.0 script = '%s(%i,%f)' % (self.filter_functions['SetEnergyFilterOffset'], technique, value) self.ExecuteSendScript(script) @@ -469,25 +468,25 @@ def SetEnergyFilterOffset(self, value): else: return -1 - def AlignEnergyFilterZeroLossPeak(self): + def AlignEnergyFilterZeroLossPeak(self) -> float: script = ' if ( %s() ) { %s Exit(1.0); } else { Exit(-1.0); }' % ( self.filter_functions['AlignEnergyFilterZeroLossPeak'], self.wait_for_filter) return self.ExecuteGetDoubleScript(script) @logwrap def GetImage(self, - processing, - height, - width, - binning, - top, - left, - bottom, - right, - exposure, - corrections, - shutter=0, - shutterDelay=0.0): + processing: str, + height: int, + width: int, + binning: int, + top: int, + left: int, + bottom: int, + right: int, + exposure: float, + corrections: int , + shutter: int = 0, + shutterDelay: float = 0.0) -> np.ndarray: arrSize = width * height @@ -528,17 +527,12 @@ def GetImage(self, message_send = Message(longargs=longargs, dblargs=dblargs) message_recv = Message(longargs=(0, 0, 0, 0, 0)) - - # attempt to solve UCLA problem by reconnecting - # if self.save_frames: - # self.reconnect() - self.ExchangeMessages(message_send, message_recv) longargs = message_recv.array['longargs'].tolist() logging.debug('GetImage longargs %s', longargs) if longargs[0] < 0: - return 1 + return 1 # FIXME arrSize = longargs[1] width = longargs[2] height = longargs[3] @@ -576,11 +570,15 @@ def GetImage(self, imArray = imArray.reshape((height, width)) return imArray - def ExecuteSendCameraObjectionFunction(self, function_name, camera_id=0): + def ExecuteSendCameraObjectionFunction(self, + function_name: str, + camera_id: int = 0) -> int: # first longargs is error code. Error if > 0 return self.ExecuteGetLongCameraObjectFunction(function_name, camera_id) - def ExecuteGetLongCameraObjectFunction(self, function_name, camera_id=0): + def ExecuteGetLongCameraObjectFunction(self, + function_name: str, + camera_id: int = 0) -> int: """ Execute DM script function that requires camera object as input and output one long integer. """ @@ -589,9 +587,11 @@ def ExecuteGetLongCameraObjectFunction(self, function_name, camera_id=0): recv_longargs_init=recv_longargs_init) if result is False: return 1 - return result.array['longargs'][0] + return int(result.array['longargs'][0]) - def ExecuteGetDoubleCameraObjectFunction(self, function_name, camera_id=0): + def ExecuteGetDoubleCameraObjectFunction(self, + function_name: str, + camera_id: int = 0) -> float: """ Execute DM script function that requires camera object as input and output double floating point number. """ @@ -600,51 +600,59 @@ def ExecuteGetDoubleCameraObjectFunction(self, function_name, camera_id=0): recv_dblargs_init=recv_dblargs_init) if result is False: return -999.0 - return result.array['dblargs'][0] + return float(result.array['dblargs'][0]) def ExecuteCameraObjectFunction(self, - function_name, - camera_id=0, - recv_longargs_init=(0,), - recv_dblargs_init=(0.0,)): + function_name: str, + camera_id: int = 0, + recv_longargs_init: Tuple = (0,), + recv_dblargs_init: Tuple = (0.0,)) -> Message: """ Execute DM script function that requires camera object as input. """ if not self.hasScriptFunction(function_name): - return False - fullcommand = "Object manager = CM_GetCameraManager();\n Object cameraList = CM_GetCameras(manager);\n Object camera = ObjectAt(cameraList,%d);\n " % ( - camera_id) - fullcommand += "%s(camera);\n" % function_name + raise NotImplementedError(function_name) + fullcommand = ["Object manager = CM_GetCameraManager();", + "Object cameraList = CM_GetCameras(manager);", + "Object camera = ObjectAt(cameraList,%d);" % camera_id, + "%s(camera);" % function_name + ] + fullcommand = "\n".join(fullcommand) result = self.ExecuteScript(fullcommand, camera_id, recv_longargs_init, recv_dblargs_init) return result - def ExecuteSendScript(self, command_line, select_camera=0): + def ExecuteSendScript(self, + command_line: str, + select_camera: int = 0) -> int: recv_longargs_init = (0,) result = self.ExecuteScript(command_line, select_camera, recv_longargs_init) # first longargs is error code. Error if > 0 - return result.array['longargs'][0] + return int(result.array['longargs'][0]) - def ExecuteGetLongScript(self, command_line, select_camera=0): + def ExecuteGetLongScript(self, + command_line: str, + select_camera: int = 0) -> int: """ Execute DM script and return the result as integer. """ - # SerialEMCCD DM TemplatePlugIn::ExecuteScript retval is a double return int(self.ExecuteGetDoubleScript(command_line, select_camera)) - def ExecuteGetDoubleScript(self, command_line, select_camera=0): + def ExecuteGetDoubleScript(self, + command_line: str, + select_camera: int = 0) -> float: """ Execute DM script that gets one double float number. """ result = self.ExecuteScript(command_line, select_camera) - return result.array['dblargs'][0] + return float(result.array['dblargs'][0]) def ExecuteScript(self, command_line: str, select_camera: int = 0, recv_longargs_init: Tuple = (0,), - recv_dblargs_init: Tuple = (0.0,)): + recv_dblargs_init: Tuple = (0.0,)) -> Message: funcCode = enum_gs['GS_ExecuteScript'] cmd_str = command_line + '\0' extra = len(cmd_str) % 4 if extra: npad = 4 - extra cmd_str = cmd_str + npad * '\0' - # send the command string as 1D longarray + # send the command string as 1D long array longarray = np.frombuffer(bytes(cmd_str, 'utf-8'), dtype=np.int32) message_send = Message(longargs=(funcCode,), boolargs=(select_camera,), longarray=longarray) message_recv = Message(longargs=recv_longargs_init, dblargs=recv_dblargs_init) @@ -658,12 +666,13 @@ def ExecuteScript(self, print('Version', g.GetDMVersion()) print('GetNumberOfCameras', g.GetNumberOfCameras()) print('GetPluginVersion', g.GetPluginVersion()) + print("SelectCamera") + g.SelectCamera(0) if not g.IsCameraInserted(0): print('InsertCamera') g.InsertCamera(0, True) print('IsCameraInserted', g.IsCameraInserted(0)) - k2params = { 'readMode': 1, # {'non-K2 cameras':-1, 'linear': 0, 'counting': 1, 'super resolution': 2} 'scaling': 1.0, @@ -676,9 +685,7 @@ def ExecuteScript(self, } def getDMVersion(g): - ''' - version: version_long, major.minor.sub - ''' + """ Return DM version details: version_long, major.minor.sub """ version_long = g.GetDMVersion() if version_long < 40000: major = 1 @@ -701,30 +708,22 @@ def getDMVersion(g): remainder = version_long - (major + 2) * 10000 minor = remainder // 100 sub = remainder % 100 - return (version_long, '%d.%d.%d' % (major, minor, sub)) - - def isDM332orUp(g): - version_id, version_string = getDMVersion(g) - if version_id and version_id >= 50302: - return True - return False + return version_long, '%d.%d.%d' % (major, minor, sub) def getCorrectionFlags(g): - ''' - Binnary Correction flag sum in GMS. See Feature #8391. - GMS3.3.2 has pre-counting correction which is superior. - SerialEM always do this correction - but Leginon 3.4 and earlier does not. + """ + Binary Correction flag sum in GMS. See Feature #8391. + GMS 3.3.2 has pre-counting correction which is superior. + SerialEM always does this correction. David M. said SerialEM default is 49 for K2 and 1 for K3. - 49 means defect,bias, and quadrant (to be the same as Ultrascan). - I don't think the latter two needs applying in counting. - ''' - if isDM332orUp(g): - return 1 # defect correction only. - else: - # keep it zero to be back compatible. - return 0 + 1 means defect correction only. + 49 means defect, bias, and quadrant (to be the same as Ultrascan). + I don't think the latter two need applying in counting. + """ + version_id, version_string = getDMVersion(g) + isDM332orUp = version_id >= 50302 + return 1 if isDM332orUp else 0 acqparams = { 'processing': 'unprocessed', # dark, dark subtracted, gain normalized From 9f79bd64ed395258d297877ba92eddd1b7769811 Mon Sep 17 00:00:00 2001 From: Grigory Sharov Date: Wed, 19 Mar 2025 17:06:58 +0000 Subject: [PATCH 04/11] add cameraname method --- pytemscript/plugins/gatan_sem_socket.py | 47 ++++++++++++++----------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/pytemscript/plugins/gatan_sem_socket.py b/pytemscript/plugins/gatan_sem_socket.py index f0eca42..e4ff5ff 100644 --- a/pytemscript/plugins/gatan_sem_socket.py +++ b/pytemscript/plugins/gatan_sem_socket.py @@ -90,7 +90,8 @@ class Message: def __init__(self, longargs=(), boolargs=(), dblargs=(), longarray=np.array([], dtype=np.int32)): - """ Initialize with the sequences of args (longs, bools, doubles) and optional long array. """ + """ Initialize with the sequences of args (longs, bools, doubles) + and optional long array. """ if len(longarray): longargs = (*longargs, len(longarray)) @@ -144,7 +145,7 @@ def __init__(self, host: str = "127.0.0.1", port: int = 48890): self.num_grab_sum = 0 self.connect() - self.script_functions = [ + script_functions = [ ('AFGetSlitState', 'GetEnergyFilter'), ('AFSetSlitState', 'SetEnergyFilter'), ('AFGetSlitWidth', 'GetEnergyFilterWidth'), @@ -165,7 +166,7 @@ def __init__(self, host: str = "127.0.0.1", port: int = 48890): ('GT_CenterZLP', 'AlignEnergyFilterZeroLossPeak'), ] self.filter_functions = {} - for name, method_name in self.script_functions: + for name, method_name in script_functions: if self.hasScriptFunction(name): self.filter_functions[method_name] = name if ('SetEnergyFilter' in self.filter_functions.keys() and @@ -246,20 +247,24 @@ def GetDMVersion(self) -> int: def GetNumberOfCameras(self) -> int: return self.GetLong('GS_GetNumberOfCameras') + def GetCameraName(self, camera_id: int) -> str: + result = self.ExecuteCameraObjectFunction("CM_GetCameraName", camera_id) + return result.array['longarray'].tobytes().decode('utf-8') + def GetPluginVersion(self) -> int: return self.GetLong('GS_GetPluginVersion') - def IsCameraInserted(self, cameraid: int) -> bool: + def IsCameraInserted(self, camera_id: int) -> bool: funcCode = enum_gs['GS_IsCameraInserted'] - message_send = Message(longargs=(funcCode, cameraid)) + message_send = Message(longargs=(funcCode, camera_id)) message_recv = Message(longargs=(0,), boolargs=(0,)) self.ExchangeMessages(message_send, message_recv) result = bool(message_recv.array['boolargs'][0]) return result - def InsertCamera(self, cameraid: int, state: int): + def InsertCamera(self, camera_id: int, state: int): funcCode = enum_gs['GS_InsertCamera'] - message_send = Message(longargs=(funcCode, cameraid), boolargs=(state,)) + message_send = Message(longargs=(funcCode, camera_id), boolargs=(state,)) message_recv = Message(longargs=(0,)) self.ExchangeMessages(message_send, message_recv) @@ -269,9 +274,9 @@ def SetReadMode(self, mode: int, scaling: float = 1.0): message_recv = Message(longargs=(0,)) self.ExchangeMessages(message_send, message_recv) - def SetShutterNormallyClosed(self, cameraid: int, shutter: int): + def SetShutterNormallyClosed(self, camera_id: int, shutter: int): funcCode = enum_gs['GS_SetShutterNormallyClosed'] - message_send = Message(longargs=(funcCode, cameraid, shutter)) + message_send = Message(longargs=(funcCode, camera_id, shutter)) message_recv = Message(longargs=(0,)) self.ExchangeMessages(message_send, message_recv) @@ -382,19 +387,19 @@ def GetFileSaveResult(self) -> Tuple: error = args[2] return numsaved, error - def SelectCamera(self, cameraid: int): + def SelectCamera(self, camera_id: int): funcCode = enum_gs['GS_SelectCamera'] - message_send = Message(longargs=(funcCode, cameraid)) + message_send = Message(longargs=(funcCode, camera_id)) message_recv = Message(longargs=(0,)) self.ExchangeMessages(message_send, message_recv) - def UpdateK2HardwareDarkReference(self, cameraid: int) -> int: + def UpdateK2HardwareDarkReference(self, camera_id: int) -> int: function_name = 'K2_updateHardwareDarkReference' - return self.ExecuteSendCameraObjectionFunction(function_name, cameraid) + return self.ExecuteSendCameraObjectionFunction(function_name, camera_id) - def PrepareDarkReference(self, cameraid: int) -> int: + def PrepareDarkReference(self, camera_id: int) -> int: function_name = 'CM_PrepareDarkReference' - return self.ExecuteSendCameraObjectionFunction(function_name, cameraid) + return self.ExecuteSendCameraObjectionFunction(function_name, camera_id) def GetEnergyFilter(self) -> float: if 'GetEnergyFilter' not in self.filter_functions: @@ -484,7 +489,7 @@ def GetImage(self, bottom: int, right: int, exposure: float, - corrections: int , + corrections: int, shutter: int = 0, shutterDelay: float = 0.0) -> np.ndarray: @@ -595,9 +600,7 @@ def ExecuteGetDoubleCameraObjectFunction(self, """ Execute DM script function that requires camera object as input and output double floating point number. """ - recv_dblargs_init = (0,) - result = self.ExecuteCameraObjectFunction(function_name, camera_id, - recv_dblargs_init=recv_dblargs_init) + result = self.ExecuteCameraObjectFunction(function_name, camera_id) if result is False: return -999.0 return float(result.array['dblargs'][0]) @@ -607,7 +610,10 @@ def ExecuteCameraObjectFunction(self, camera_id: int = 0, recv_longargs_init: Tuple = (0,), recv_dblargs_init: Tuple = (0.0,)) -> Message: - """ Execute DM script function that requires camera object as input. """ + """ Execute DM script function that requires camera object as input. + See examples at http://www.dmscripting.com/files/CameraScriptDocumentation.txt + and http://www.dmscripting.com/tutorial_microscope_commands.html + """ if not self.hasScriptFunction(function_name): raise NotImplementedError(function_name) fullcommand = ["Object manager = CM_GetCameraManager();", @@ -668,6 +674,7 @@ def ExecuteScript(self, print('GetPluginVersion', g.GetPluginVersion()) print("SelectCamera") g.SelectCamera(0) + print("GetCameraName", g.GetCameraName(0)) if not g.IsCameraInserted(0): print('InsertCamera') g.InsertCamera(0, True) From 3b3353801b331d8d063eae502b65a72b1800ae78 Mon Sep 17 00:00:00 2001 From: Grigory Sharov Date: Wed, 19 Mar 2025 21:15:31 +0000 Subject: [PATCH 05/11] reogranize plugin code --- pytemscript/plugins/gatan_sem_socket.py | 753 ------------------ .../plugins/serialem_ccd/base_socket.py | 229 ++++++ pytemscript/plugins/serialem_ccd/plugin.py | 368 +++++++++ pytemscript/plugins/serialem_ccd/utils.py | 128 +++ tests/test_sem_ccd.py | 94 +++ 5 files changed, 819 insertions(+), 753 deletions(-) delete mode 100644 pytemscript/plugins/gatan_sem_socket.py create mode 100644 pytemscript/plugins/serialem_ccd/base_socket.py create mode 100644 pytemscript/plugins/serialem_ccd/plugin.py create mode 100644 pytemscript/plugins/serialem_ccd/utils.py create mode 100644 tests/test_sem_ccd.py diff --git a/pytemscript/plugins/gatan_sem_socket.py b/pytemscript/plugins/gatan_sem_socket.py deleted file mode 100644 index e4ff5ff..0000000 --- a/pytemscript/plugins/gatan_sem_socket.py +++ /dev/null @@ -1,753 +0,0 @@ -# The Leginon software is Copyright 2003 -# The Scripps Research Institute, La Jolla, CA -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# The code below is a modified version of: https://github.com/leginon-org/leginon/blob/myami-python3/pyscope/gatansocket.py -# This plugin needs the SERIALEMCCD plugin to be installed in DigitalMicrograph -# The instructions can be found at: https://bio3d.colorado.edu/SerialEM/hlp/html/setting_up_serialem.htm -# -# On the computer running DM, define environment variables: -# SERIALEMCCD_PORT=48890 -# [optional] SERIALEMCCD_DEBUG=1 - -import logging -import functools -import socket -from typing import Tuple, Optional -import numpy as np - -# enum function codes as in https://github.com/mastcu/SerialEMCCD/blob/master/SocketPathway.cpp -# need to match exactly both in number and order -enum_gs = [ - 'GS_ExecuteScript', - 'GS_SetDebugMode', - 'GS_SetDMVersion', - 'GS_SetCurrentCamera', - 'GS_QueueScript', - 'GS_GetAcquiredImage', - 'GS_GetDarkReference', - 'GS_GetGainReference', - 'GS_SelectCamera', - 'GS_SetReadMode', - 'GS_GetNumberOfCameras', - 'GS_IsCameraInserted', - 'GS_InsertCamera', - 'GS_GetDMVersion', - 'GS_GetDMCapabilities', - 'GS_SetShutterNormallyClosed', - 'GS_SetNoDMSettling', - 'GS_GetDSProperties', - 'GS_AcquireDSImage', - 'GS_ReturnDSChannel', - 'GS_StopDSAcquisition', - 'GS_CheckReferenceTime', - 'GS_SetK2Parameters', - 'GS_ChunkHandshake', # deleted in the new plugin? - 'GS_SetupFileSaving', - 'GS_GetFileSaveResult', - 'GS_SetupFileSaving2', - 'GS_GetDefectList', - 'GS_SetK2Parameters2', - 'GS_StopContinuousCamera', - 'GS_GetPluginVersion', - 'GS_GetLastError', - 'GS_FreeK2GainReference', - 'GS_IsGpuAvailable', - 'GS_SetupFrameAligning', - 'GS_FrameAlignResults', - 'GS_ReturnDeferredSum', - 'GS_MakeAlignComFile', - 'GS_WaitUntilReady', - 'GS_GetLastDoseRate', - 'GS_SaveFrameMdoc', - 'GS_GetDMVersionAndBuild', - 'GS_GetTiltSumProperties', -] -# lookup table of function name to function code, starting with 1 -enum_gs = {item: index for index, item in enumerate(enum_gs, 1)} - -## C "long" -> numpy "int32" -ARGS_BUFFER_SIZE = 1024 -MAX_LONG_ARGS = 16 -MAX_DBL_ARGS = 8 -MAX_BOOL_ARGS = 8 -sArgsBuffer = np.zeros(ARGS_BUFFER_SIZE, dtype=np.byte) - - -class Message: - """ Information packet to send and receive on the socket. """ - def __init__(self, - longargs=(), boolargs=(), dblargs=(), - longarray=np.array([], dtype=np.int32)): - """ Initialize with the sequences of args (longs, bools, doubles) - and optional long array. """ - - if len(longarray): - longargs = (*longargs, len(longarray)) - - self.dtype = [ - ('size', np.intc, (1,)), - ('longargs', np.int32, (len(longargs),)), - ('boolargs', np.int32, (len(boolargs),)), - ('dblargs', np.double, (len(dblargs),)), - ('longarray', np.int32, (len(longarray),)), - ] - self.array = np.empty((), dtype=self.dtype) - self.array['size'] = np.array([self.array.nbytes], dtype=np.intc) - self.array['longargs'] = np.array(longargs, dtype=np.int32) - self.array['boolargs'] = np.array(boolargs, dtype=np.int32) - self.array['dblargs'] = np.array(dblargs, dtype=np.double) - self.array['longarray'] = longarray - - def pack(self) -> memoryview: - """ Serialize the data. """ - packed = self.array.nbytes - if packed > ARGS_BUFFER_SIZE: - raise RuntimeError('Message packet size %d is larger than maximum %d' % (packed, ARGS_BUFFER_SIZE)) - return self.array.data - - def unpack(self, buf: bytes) -> None: - """ Unpack buffer into our data structure. """ - self.array = np.frombuffer(buf, dtype=self.dtype)[0] - - -def logwrap(func): - """ Decorator to log socket send and recv calls. """ - @functools.wraps(func) - def wrapper(*args, **kwargs): - logging.debug('%s\t%r\t%r', func.__name__, args, kwargs) - try: - return func(*args, **kwargs) - except Exception as e: - logging.error('EXCEPTION: %s', e) - raise - - return wrapper - - -class GatanSocket: - def __init__(self, host: str = "127.0.0.1", port: int = 48890): - self.sock = None - self.host = host - self.port = port - self.save_frames = False - self.num_grab_sum = 0 - self.connect() - - script_functions = [ - ('AFGetSlitState', 'GetEnergyFilter'), - ('AFSetSlitState', 'SetEnergyFilter'), - ('AFGetSlitWidth', 'GetEnergyFilterWidth'), - ('AFSetSlitWidth', 'SetEnergyFilterWidth'), - ('AFDoAlignZeroLoss', 'AlignEnergyFilterZeroLossPeak'), - ('IFCGetSlitState', 'GetEnergyFilter'), - ('IFCSetSlitState', 'SetEnergyFilter'), - ('IFCGetSlitWidth', 'GetEnergyFilterWidth'), - ('IFCSetSlitWidth', 'SetEnergyFilterWidth'), - ('IFCDoAlignZeroLoss', 'AlignEnergyFilterZeroLossPeak'), - ('IFGetSlitIn', 'GetEnergyFilter'), - ('IFSetSlitIn', 'SetEnergyFilter'), - ('IFGetEnergyLoss', 'GetEnergyFilterOffset'), - ('IFSetEnergyOffset', 'SetEnergyFilterOffset'), # wjr this was IFSetEnergyLoss - ('IFGetMaximumSlitWidth', 'GetEnergyFilterWidthMax'), - ('IFGetSlitWidth', 'GetEnergyFilterWidth'), - ('IFSetSlitWidth', 'SetEnergyFilterWidth'), - ('GT_CenterZLP', 'AlignEnergyFilterZeroLossPeak'), - ] - self.filter_functions = {} - for name, method_name in script_functions: - if self.hasScriptFunction(name): - self.filter_functions[method_name] = name - if ('SetEnergyFilter' in self.filter_functions.keys() and - self.filter_functions['SetEnergyFilter'] == 'IFSetSlitIn'): - self.wait_for_filter = 'IFWaitForFilter();' - else: - self.wait_for_filter = '' - - def hasScriptFunction(self, name: str) -> bool: - script = 'if ( DoesFunctionExist("%s") ) { Exit(1.0); } else { Exit(-1.0); }' % name - result = self.ExecuteGetDoubleScript(script) - return result > 0.0 - - def connect(self) -> None: - # recommended by Gatan to use localhost IP to avoid using tcp - self.sock = socket.create_connection((self.host, self.port)) - - def disconnect(self) -> None: - self.sock.shutdown(socket.SHUT_RDWR) - self.sock.close() - - def reconnect(self) -> None: - self.disconnect() - self.connect() - - @logwrap - def send_data(self, data: memoryview) -> None: - self.sock.sendall(data) - - @logwrap - def recv_data(self, n: int) -> bytes: - return self.sock.recv(n) - - def ExchangeMessages(self, message_send: Message, - message_recv: Optional[Message] = None): - self.send_data(message_send.pack()) - if message_recv is None: - return - recv_buffer = message_recv.pack() - recv_len = recv_buffer.nbytes - total_recv = 0 - parts = [] - while total_recv < recv_len: - remain = recv_len - total_recv - new_recv = self.recv_data(remain) - parts.append(new_recv) - total_recv += len(new_recv) - buf = b''.join(parts) - message_recv.unpack(buf) - ## log the error code from received message - sendargs = message_send.array['longargs'] - recvargs = message_recv.array['longargs'] - logging.debug('Func: %d, Code: %d', sendargs[0], recvargs[0]) - - def GetLong(self, funcName: str) -> int: - """ Common class of function that gets a single long """ - funcCode = enum_gs[funcName] - message_send = Message(longargs=(funcCode,)) - # First recieved message longargs is error code - message_recv = Message(longargs=(0, 0)) - self.ExchangeMessages(message_send, message_recv) - result = int(message_recv.array['longargs'][1]) - return result - - def SendLongGetLong(self, funcName: str, longarg) -> int: - """ Common class of function with one long arg that returns a single long """ - funcCode = enum_gs[funcName] - message_send = Message(longargs=(funcCode, longarg)) - # First recieved message longargs is error code - message_recv = Message(longargs=(0, 0)) - self.ExchangeMessages(message_send, message_recv) - result = int(message_recv.array['longargs'][1]) - return result - - def GetDMVersion(self) -> int: - return self.GetLong('GS_GetDMVersion') - - def GetNumberOfCameras(self) -> int: - return self.GetLong('GS_GetNumberOfCameras') - - def GetCameraName(self, camera_id: int) -> str: - result = self.ExecuteCameraObjectFunction("CM_GetCameraName", camera_id) - return result.array['longarray'].tobytes().decode('utf-8') - - def GetPluginVersion(self) -> int: - return self.GetLong('GS_GetPluginVersion') - - def IsCameraInserted(self, camera_id: int) -> bool: - funcCode = enum_gs['GS_IsCameraInserted'] - message_send = Message(longargs=(funcCode, camera_id)) - message_recv = Message(longargs=(0,), boolargs=(0,)) - self.ExchangeMessages(message_send, message_recv) - result = bool(message_recv.array['boolargs'][0]) - return result - - def InsertCamera(self, camera_id: int, state: int): - funcCode = enum_gs['GS_InsertCamera'] - message_send = Message(longargs=(funcCode, camera_id), boolargs=(state,)) - message_recv = Message(longargs=(0,)) - self.ExchangeMessages(message_send, message_recv) - - def SetReadMode(self, mode: int, scaling: float = 1.0): - funcCode = enum_gs['GS_SetReadMode'] - message_send = Message(longargs=(funcCode, mode), dblargs=(scaling,)) - message_recv = Message(longargs=(0,)) - self.ExchangeMessages(message_send, message_recv) - - def SetShutterNormallyClosed(self, camera_id: int, shutter: int): - funcCode = enum_gs['GS_SetShutterNormallyClosed'] - message_send = Message(longargs=(funcCode, camera_id, shutter)) - message_recv = Message(longargs=(0,)) - self.ExchangeMessages(message_send, message_recv) - - @logwrap - def SetK2Parameters(self, - readMode: int, - scaling: float = 1.0, - hardwareProc: int = 1, - doseFrac: bool = False, - frameTime: float = 0.25, - alignFrames: bool = False, - saveFrames: bool = False, - filt: str = '', - useCds: bool = False): - funcCode = enum_gs['GS_SetK2Parameters2'] - # rotation and flip for non-frame saving image. It is the same definition - # as in SetFileSaving2. If set to 0, it takes the value from GMS - rotationFlip = 0 - self.save_frames = saveFrames - - flags = 0 - flags += int(useCds) * 2 ** 6 - # settings of unused flags - # anti_alias - reducedSizes = 0 - fullSizes = 0 - - # filter name - filt_str = filt + '\0' - extra = len(filt_str) % 4 - if extra: - npad = 4 - extra - filt_str = filt_str + npad * '\0' - longarray = np.frombuffer(bytes(filt_str, 'utf-8'), dtype=np.int32) - - longs = [ - funcCode, - readMode, - hardwareProc, - rotationFlip, - flags, - ] - bools = [ - doseFrac, - alignFrames, - saveFrames, - ] - doubles = [ - scaling, - frameTime, - reducedSizes, - fullSizes, - 0.0, # dummy3 - 0.0, # dummy4 - ] - - message_send = Message(longargs=longs, boolargs=bools, dblargs=doubles, longarray=longarray) - message_recv = Message(longargs=(0,)) # just return code - self.ExchangeMessages(message_send, message_recv) - - def setNumGrabSum(self, earlyReturnFrameCount: int, earlyReturnRamGrabs: int): - # pack RamGrabs and earlyReturnFrameCount in one double - self.num_grab_sum = 2**16 * earlyReturnRamGrabs + earlyReturnFrameCount - - def getNumGrabSum(self) -> int: - return self.num_grab_sum - - @logwrap - def SetupFileSaving(self, - rotationFlip, - dirname: str, - rootname: str, - filePerImage: int, - doEarlyReturn: bool = False, - earlyReturnFrameCount: int = 0, - earlyReturnRamGrabs: int = 0, - lzwtiff: bool = False) -> None: - pixelSize = 1.0 - self.setNumGrabSum(earlyReturnFrameCount, earlyReturnRamGrabs) - if self.save_frames and (doEarlyReturn or lzwtiff): - # early return flag - flag = 128 * int(doEarlyReturn) + 8 * int(lzwtiff) - numGrabSum = self.getNumGrabSum() - # set values to pass - longs = [enum_gs['GS_SetupFileSaving2'], rotationFlip, flag, ] - dbls = [pixelSize, numGrabSum, 0., 0., 0., ] - else: - longs = [enum_gs['GS_SetupFileSaving'], rotationFlip, ] - dbls = [pixelSize, ] - bools = [filePerImage, ] - names_str = dirname + '\0' + rootname + '\0' - extra = len(names_str) % 4 - if extra: - npad = 4 - extra - names_str = names_str + npad * '\0' - longarray = np.frombuffer(bytes(names_str, 'utf-8'), dtype=np.int32) - message_send = Message(longargs=longs, boolargs=bools, dblargs=dbls, longarray=longarray) - message_recv = Message(longargs=(0, 0)) - self.ExchangeMessages(message_send, message_recv) - - def GetFileSaveResult(self) -> Tuple: - #longs = [enum_gs['GS_GetFileSaveResult'], rotationFlip] - message_send = Message(longargs=(enum_gs['GS_GetFileSaveResult'],))#, boolargs=bools, dblargs=dbls, longarray=longarray) - message_recv = Message(longargs=(0, 0, 0)) - self.ExchangeMessages(message_send, message_recv) - args = message_recv.array['longargs'] - numsaved = args[1] - error = args[2] - return numsaved, error - - def SelectCamera(self, camera_id: int): - funcCode = enum_gs['GS_SelectCamera'] - message_send = Message(longargs=(funcCode, camera_id)) - message_recv = Message(longargs=(0,)) - self.ExchangeMessages(message_send, message_recv) - - def UpdateK2HardwareDarkReference(self, camera_id: int) -> int: - function_name = 'K2_updateHardwareDarkReference' - return self.ExecuteSendCameraObjectionFunction(function_name, camera_id) - - def PrepareDarkReference(self, camera_id: int) -> int: - function_name = 'CM_PrepareDarkReference' - return self.ExecuteSendCameraObjectionFunction(function_name, camera_id) - - def GetEnergyFilter(self) -> float: - if 'GetEnergyFilter' not in self.filter_functions: - return -1.0 - script = 'if ( %s() ) { Exit(1.0); } else { Exit(-1.0); }' % (self.filter_functions['GetEnergyFilter'],) - return self.ExecuteGetDoubleScript(script) - - def SetEnergyFilter(self, value: bool = False) -> int: - if 'SetEnergyFilter' not in self.filter_functions: - return -1 - if value: - i = 1 - else: - i = 0 - script = '%s(%d); %s' % (self.filter_functions['SetEnergyFilter'], i, self.wait_for_filter) - return self.ExecuteSendScript(script) - - def GetEnergyFilterWidthMax(self) -> float: - if 'GetEnergyFilterWidthMax' not in self.filter_functions: - return -1.0 - script = 'Exit(%s())' % (self.filter_functions['GetEnergyFilterWidthMax'],) - return self.ExecuteGetDoubleScript(script) - - def GetEnergyFilterWidth(self) -> float: - if 'GetEnergyFilterWidth' not in self.filter_functions: - return -1.0 - script = 'Exit(%s())' % (self.filter_functions['GetEnergyFilterWidth'],) - return self.ExecuteGetDoubleScript(script) - - def SetEnergyFilterWidth(self, value: float) -> int: - if 'SetEnergyFilterWidth' not in self.filter_functions: - return -1 - script = 'if ( %s(%f) ) { Exit(1.0); } else { Exit(-1.0); }' % ( - self.filter_functions['SetEnergyFilterWidth'], value) - return self.ExecuteSendScript(script) - - def GetEnergyFilterOffset(self) -> float: - if 'GetEnergyFilterOffset' not in self.filter_functions: - return 0.0 - script = 'Exit(%s())' % (self.filter_functions['GetEnergyFilterOffset'],) - return self.ExecuteGetDoubleScript(script) - - def SetEnergyFilterOffset(self, value: float) -> float: - """ - wjr changing this to use the Gatan function IFSetEnergyOffset, which needs a technique and a value - GMS 3.32,function apparently added in GMS 3.2. Later versions will need to be checked - technique 0: instrument (not available) - technique 1: prism offset (confusing because -10 is 10) - technique 2: HT offset (has no effect on BioQuantum, but HT offset in DM will change) - technique 3: drift tube ( this seems best since the value is consistant with direction) - technique 4: prism adjust (confusing because -10 is 10 and it does not count when checking the energy loss value) - note: the Gatan function being called is a void, so removed the boolean logic used for most other functions - """ - technique = 3 # hard code to drift tube for now - if 'SetEnergyFilterOffset' not in self.filter_functions: - return -1.0 - script = '%s(%i,%f)' % (self.filter_functions['SetEnergyFilterOffset'], technique, value) - self.ExecuteSendScript(script) - # return 1 - # or better to - newvalue = self.GetEnergyFilterOffset() # ? but wastes time - if value == newvalue: - return 1 - else: - technique = 2 # reset the HT offfset to 0, sometimes this gets set - script = '%s(%i,%f)' % (self.filter_functions['SetEnergyFilterOffset'], technique, 0) - self.ExecuteSendScript(script) - newvalue = self.GetEnergyFilterOffset() - if value == newvalue: - return 1 - else: - return -1 - - def AlignEnergyFilterZeroLossPeak(self) -> float: - script = ' if ( %s() ) { %s Exit(1.0); } else { Exit(-1.0); }' % ( - self.filter_functions['AlignEnergyFilterZeroLossPeak'], self.wait_for_filter) - return self.ExecuteGetDoubleScript(script) - - @logwrap - def GetImage(self, - processing: str, - height: int, - width: int, - binning: int, - top: int, - left: int, - bottom: int, - right: int, - exposure: float, - corrections: int, - shutter: int = 0, - shutterDelay: float = 0.0) -> np.ndarray: - - arrSize = width * height - - # TODO: need to figure out what these should be - divideBy2 = 0 - settling = 0.0 - - # prepare args for message - if processing == 'dark': - longargs = [enum_gs['GS_GetDarkReference']] - else: - longargs = [enum_gs['GS_GetAcquiredImage']] - longargs.extend([ - arrSize, # pixels in the image - width, height, - ]) - if processing == 'unprocessed': - longargs.append(0) - elif processing == 'dark subtracted': - longargs.append(1) - elif processing == 'gain normalized': - longargs.append(2) - longargs.extend([ - binning, - top, left, bottom, right, - shutter, - ]) - if processing != 'dark': - longargs.append(shutterDelay) - longargs.extend([ - divideBy2, - corrections, - ]) - dblargs = [ - exposure, - settling, - ] - - message_send = Message(longargs=longargs, dblargs=dblargs) - message_recv = Message(longargs=(0, 0, 0, 0, 0)) - self.ExchangeMessages(message_send, message_recv) - - longargs = message_recv.array['longargs'].tolist() - logging.debug('GetImage longargs %s', longargs) - if longargs[0] < 0: - return 1 # FIXME - arrSize = longargs[1] - width = longargs[2] - height = longargs[3] - numChunks = longargs[4] - bytesPerPixel = 2 # depends on the results formated =uint16 - numBytes = arrSize * bytesPerPixel - chunkSize = (numBytes + numChunks - 1) // numChunks - imArray = np.zeros((height * width,), np.uint16) - received = 0 - remain = numBytes - index = 0 - logging.debug('chunk size %d', chunkSize) - for chunk in range(numChunks): - recv_bytes = b'' - # send chunk handshake for all but the first chunk - if chunk: - message_send = Message(longargs=(enum_gs['GS_ChunkHandshake'],)) - self.ExchangeMessages(message_send) - thisChunkSize = min(remain, chunkSize) - chunkReceived = 0 - chunkRemain = thisChunkSize - while chunkRemain: - new_recv = self.recv_data(chunkRemain) - len_recv = len(new_recv) - recv_bytes += new_recv - chunkReceived += len_recv - chunkRemain -= len_recv - remain -= len_recv - received += len_recv - last_index = int(index) - logging.debug('chunk bytes length %d', len(recv_bytes)) - index = chunkSize * (chunk + 1) // bytesPerPixel - logging.debug('array index range to fill %d:%d', last_index, index) - imArray[last_index:index] = np.frombuffer(recv_bytes, dtype=np.uint16) - imArray = imArray.reshape((height, width)) - return imArray - - def ExecuteSendCameraObjectionFunction(self, - function_name: str, - camera_id: int = 0) -> int: - # first longargs is error code. Error if > 0 - return self.ExecuteGetLongCameraObjectFunction(function_name, camera_id) - - def ExecuteGetLongCameraObjectFunction(self, - function_name: str, - camera_id: int = 0) -> int: - """ Execute DM script function that requires camera object - as input and output one long integer. - """ - recv_longargs_init = (0,) - result = self.ExecuteCameraObjectFunction(function_name, camera_id, - recv_longargs_init=recv_longargs_init) - if result is False: - return 1 - return int(result.array['longargs'][0]) - - def ExecuteGetDoubleCameraObjectFunction(self, - function_name: str, - camera_id: int = 0) -> float: - """ Execute DM script function that requires camera object - as input and output double floating point number. - """ - result = self.ExecuteCameraObjectFunction(function_name, camera_id) - if result is False: - return -999.0 - return float(result.array['dblargs'][0]) - - def ExecuteCameraObjectFunction(self, - function_name: str, - camera_id: int = 0, - recv_longargs_init: Tuple = (0,), - recv_dblargs_init: Tuple = (0.0,)) -> Message: - """ Execute DM script function that requires camera object as input. - See examples at http://www.dmscripting.com/files/CameraScriptDocumentation.txt - and http://www.dmscripting.com/tutorial_microscope_commands.html - """ - if not self.hasScriptFunction(function_name): - raise NotImplementedError(function_name) - fullcommand = ["Object manager = CM_GetCameraManager();", - "Object cameraList = CM_GetCameras(manager);", - "Object camera = ObjectAt(cameraList,%d);" % camera_id, - "%s(camera);" % function_name - ] - fullcommand = "\n".join(fullcommand) - result = self.ExecuteScript(fullcommand, camera_id, recv_longargs_init, - recv_dblargs_init) - return result - - def ExecuteSendScript(self, - command_line: str, - select_camera: int = 0) -> int: - recv_longargs_init = (0,) - result = self.ExecuteScript(command_line, select_camera, recv_longargs_init) - # first longargs is error code. Error if > 0 - return int(result.array['longargs'][0]) - - def ExecuteGetLongScript(self, - command_line: str, - select_camera: int = 0) -> int: - """ Execute DM script and return the result as integer. """ - return int(self.ExecuteGetDoubleScript(command_line, select_camera)) - - def ExecuteGetDoubleScript(self, - command_line: str, - select_camera: int = 0) -> float: - """ Execute DM script that gets one double float number. """ - result = self.ExecuteScript(command_line, select_camera) - return float(result.array['dblargs'][0]) - - def ExecuteScript(self, - command_line: str, - select_camera: int = 0, - recv_longargs_init: Tuple = (0,), - recv_dblargs_init: Tuple = (0.0,)) -> Message: - funcCode = enum_gs['GS_ExecuteScript'] - cmd_str = command_line + '\0' - extra = len(cmd_str) % 4 - if extra: - npad = 4 - extra - cmd_str = cmd_str + npad * '\0' - # send the command string as 1D long array - longarray = np.frombuffer(bytes(cmd_str, 'utf-8'), dtype=np.int32) - message_send = Message(longargs=(funcCode,), boolargs=(select_camera,), longarray=longarray) - message_recv = Message(longargs=recv_longargs_init, dblargs=recv_dblargs_init) - self.ExchangeMessages(message_send, message_recv) - return message_recv - - -if __name__ == '__main__': - g = GatanSocket(host="192.168.71.2") - print(g) - print('Version', g.GetDMVersion()) - print('GetNumberOfCameras', g.GetNumberOfCameras()) - print('GetPluginVersion', g.GetPluginVersion()) - print("SelectCamera") - g.SelectCamera(0) - print("GetCameraName", g.GetCameraName(0)) - if not g.IsCameraInserted(0): - print('InsertCamera') - g.InsertCamera(0, True) - print('IsCameraInserted', g.IsCameraInserted(0)) - - k2params = { - 'readMode': 1, # {'non-K2 cameras':-1, 'linear': 0, 'counting': 1, 'super resolution': 2} - 'scaling': 1.0, - 'hardwareProc': 1, #{'none': 0, 'dark': 2, 'gain': 4, 'dark+gain': 6} - 'doseFrac': False, - 'frameTime': 0.25, - 'alignFrames': False, - 'saveFrames': False, - 'filt': 'None' - } - - def getDMVersion(g): - """ Return DM version details: version_long, major.minor.sub """ - version_long = g.GetDMVersion() - if version_long < 40000: - major = 1 - minor = None - sub = None - if version_long >= 31100: - # 2.3.0 gives an odd number of 31100 - major = 2 - minor = 3 - sub = 0 - version_long = 40300 - elif version_long == 40000: - # minor version can be 0 or 1 in this case - # but likely 1 since we have not used this module until k2 is around - major = 2 - minor = 1 - sub = 0 - else: - major = version_long // 10000 - 2 - remainder = version_long - (major + 2) * 10000 - minor = remainder // 100 - sub = remainder % 100 - - return version_long, '%d.%d.%d' % (major, minor, sub) - - def getCorrectionFlags(g): - """ - Binary Correction flag sum in GMS. See Feature #8391. - GMS 3.3.2 has pre-counting correction which is superior. - SerialEM always does this correction. - David M. said SerialEM default is 49 for K2 and 1 for K3. - 1 means defect correction only. - 49 means defect, bias, and quadrant (to be the same as Ultrascan). - I don't think the latter two need applying in counting. - """ - version_id, version_string = getDMVersion(g) - isDM332orUp = version_id >= 50302 - return 1 if isDM332orUp else 0 - - acqparams = { - 'processing': 'unprocessed', # dark, dark subtracted, gain normalized - 'height': 2048, - 'width': 2048, - 'binning': 1, - 'top': 0, - 'left': 0, - 'bottom': 2048, - 'right': 2048, - 'exposure': 1.0, - 'corrections': getCorrectionFlags(g), - 'shutter': 0, - 'shutterDelay': 0.0, - } - - #g.SetK2Parameters(**k2params) - g.SetReadMode(-1) - image = g.GetImage(**acqparams) - print("Got image array:", image.dtype, image.shape) diff --git a/pytemscript/plugins/serialem_ccd/base_socket.py b/pytemscript/plugins/serialem_ccd/base_socket.py new file mode 100644 index 0000000..bc66cb5 --- /dev/null +++ b/pytemscript/plugins/serialem_ccd/base_socket.py @@ -0,0 +1,229 @@ +# The Leginon software is Copyright 2003 +# The Scripps Research Institute, La Jolla, CA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# The code below is a modified version of: https://github.com/leginon-org/leginon/blob/myami-python3/pyscope/gatansocket.py + +import logging +import socket +import numpy as np +from typing import Tuple, Optional + +from pytemscript.plugins.serialem_ccd.utils import Message, logwrap, enum_gs +from pytemscript.utils.misc import setup_logging + + +class BaseSocket: + def __init__(self, + host: str = "127.0.0.1", + port: int = 48890, + debug: bool = False): + self.sock = None + self.host = host + self.port = port + self.save_frames = False + self.num_grab_sum = 0 + + setup_logging("semccd_socket_client.log", prefix="[SEMCCD]", debug=debug) + self.connect() + + script_functions = [ + ('AFGetSlitState', 'GetEnergyFilter'), + ('AFSetSlitState', 'SetEnergyFilter'), + ('AFGetSlitWidth', 'GetEnergyFilterWidth'), + ('AFSetSlitWidth', 'SetEnergyFilterWidth'), + ('AFDoAlignZeroLoss', 'AlignEnergyFilterZeroLossPeak'), + ('IFCGetSlitState', 'GetEnergyFilter'), + ('IFCSetSlitState', 'SetEnergyFilter'), + ('IFCGetSlitWidth', 'GetEnergyFilterWidth'), + ('IFCSetSlitWidth', 'SetEnergyFilterWidth'), + ('IFCDoAlignZeroLoss', 'AlignEnergyFilterZeroLossPeak'), + ('IFGetSlitIn', 'GetEnergyFilter'), + ('IFSetSlitIn', 'SetEnergyFilter'), + ('IFGetEnergyLoss', 'GetEnergyFilterOffset'), + ('IFSetEnergyOffset', 'SetEnergyFilterOffset'), # wjr this was IFSetEnergyLoss + ('IFGetMaximumSlitWidth', 'GetEnergyFilterWidthMax'), + ('IFGetSlitWidth', 'GetEnergyFilterWidth'), + ('IFSetSlitWidth', 'SetEnergyFilterWidth'), + ('GT_CenterZLP', 'AlignEnergyFilterZeroLossPeak'), + ] + self.filter_functions = {} + for name, method_name in script_functions: + if self.has_script_function(name): + self.filter_functions[method_name] = name + if ('SetEnergyFilter' in self.filter_functions.keys() and + self.filter_functions['SetEnergyFilter'] == 'IFSetSlitIn'): + self.wait_for_filter = 'IFWaitForFilter();' + else: + self.wait_for_filter = '' + + def has_script_function(self, name: str) -> bool: + script = 'if ( DoesFunctionExist("%s") ) { Exit(1.0); } else { Exit(-1.0); }' % name + result = self.ExecuteGetDoubleScript(script) + return result > 0.0 + + def connect(self) -> None: + # recommended by Gatan to use localhost IP to avoid using tcp + try: + self.sock = socket.create_connection((self.host, self.port)) + except Exception as e: + raise RuntimeError("Error communicating with SerialEMCCD socket server: %s" % e) + + def disconnect(self) -> None: + """ Disconnect from the remote server. """ + self.sock.shutdown(socket.SHUT_RDWR) + self.sock.close() + + def reconnect(self) -> None: + self.disconnect() + self.connect() + + @logwrap + def send_data(self, data: memoryview) -> None: + self.sock.sendall(data) + + @logwrap + def recv_data(self, n: int) -> bytes: + return self.sock.recv(n) + + def exchange_messages(self, message_send: Message, + message_recv: Optional[Message] = None): + self.send_data(message_send.pack()) + if message_recv is None: + return + recv_buffer = message_recv.pack() + recv_len = recv_buffer.nbytes + total_recv = 0 + parts = [] + while total_recv < recv_len: + remain = recv_len - total_recv + new_recv = self.recv_data(remain) + parts.append(new_recv) + total_recv += len(new_recv) + buf = b''.join(parts) + message_recv.unpack(buf) + ## log the error code from received message + sendargs = message_send.array['longargs'] + recvargs = message_recv.array['longargs'] + logging.debug('Func: %d, Code: %d', sendargs[0], recvargs[0]) + + def get_long(self, funcName: str) -> int: + """ Common class of function that gets a single long """ + funcCode = enum_gs[funcName] + message_send = Message(longargs=(funcCode,)) + # First recieved message longargs is error code + message_recv = Message(longargs=(0, 0)) + self.exchange_messages(message_send, message_recv) + result = int(message_recv.array['longargs'][1]) + return result + + def send_long_get_long(self, funcName: str, longarg) -> int: + """ Common class of function with one long arg that returns a single long """ + funcCode = enum_gs[funcName] + message_send = Message(longargs=(funcCode, longarg)) + # First recieved message longargs is error code + message_recv = Message(longargs=(0, 0)) + self.exchange_messages(message_send, message_recv) + result = int(message_recv.array['longargs'][1]) + return result + + def ExecuteSendCameraObjectionFunction(self, + function_name: str, + camera_id: int = 0) -> int: + # first longargs is error code. Error if > 0 + return self.ExecuteGetLongCameraObjectFunction(function_name, camera_id) + + def ExecuteGetLongCameraObjectFunction(self, + function_name: str, + camera_id: int = 0) -> int: + """ Execute DM script function that requires camera object + as input and output one long integer. + """ + recv_longargs_init = (0,) + result = self.ExecuteCameraObjectFunction(function_name, camera_id, + recv_longargs_init=recv_longargs_init) + if result is False: + return 1 + return int(result.array['longargs'][0]) + + def ExecuteGetDoubleCameraObjectFunction(self, + function_name: str, + camera_id: int = 0) -> float: + """ Execute DM script function that requires camera object + as input and output double floating point number. + """ + result = self.ExecuteCameraObjectFunction(function_name, camera_id) + if result is False: + return -999.0 + return float(result.array['dblargs'][0]) + + def ExecuteCameraObjectFunction(self, + function_name: str, + camera_id: int = 0, + recv_longargs_init: Tuple = (0,), + recv_dblargs_init: Tuple = (0.0,)) -> Message: + """ Execute DM script function that requires camera object as input. + See examples at http://www.dmscripting.com/files/CameraScriptDocumentation.txt + and http://www.dmscripting.com/tutorial_microscope_commands.html + """ + if not self.has_script_function(function_name): + raise NotImplementedError(function_name) + fullcommand = ["Object manager = CM_GetCameraManager();", + "Object cameraList = CM_GetCameras(manager);", + "Object camera = ObjectAt(cameraList,%d);" % camera_id, + "%s(camera);" % function_name + ] + fullcommand = "\n".join(fullcommand) + result = self.ExecuteScript(fullcommand, camera_id, recv_longargs_init, + recv_dblargs_init) + return result + + def ExecuteSendScript(self, + command_line: str, + select_camera: int = 0) -> int: + recv_longargs_init = (0,) + result = self.ExecuteScript(command_line, select_camera, recv_longargs_init) + # first longargs is error code. Error if > 0 + return int(result.array['longargs'][0]) + + def ExecuteGetLongScript(self, + command_line: str, + select_camera: int = 0) -> int: + """ Execute DM script and return the result as integer. """ + return int(self.ExecuteGetDoubleScript(command_line, select_camera)) + + def ExecuteGetDoubleScript(self, + command_line: str, + select_camera: int = 0) -> float: + """ Execute DM script that gets one double float number. """ + result = self.ExecuteScript(command_line, select_camera) + return float(result.array['dblargs'][0]) + + def ExecuteScript(self, + command_line: str, + select_camera: int = 0, + recv_longargs_init: Tuple = (0,), + recv_dblargs_init: Tuple = (0.0,)) -> Message: + funcCode = enum_gs['GS_ExecuteScript'] + cmd_str = command_line + '\0' + extra = len(cmd_str) % 4 + if extra: + npad = 4 - extra + cmd_str = cmd_str + npad * '\0' + # send the command string as 1D long array + longarray = np.frombuffer(bytes(cmd_str, 'utf-8'), dtype=np.int32) + message_send = Message(longargs=(funcCode,), boolargs=(select_camera,), longarray=longarray) + message_recv = Message(longargs=recv_longargs_init, dblargs=recv_dblargs_init) + self.exchange_messages(message_send, message_recv) + return message_recv diff --git a/pytemscript/plugins/serialem_ccd/plugin.py b/pytemscript/plugins/serialem_ccd/plugin.py new file mode 100644 index 0000000..47ea1bb --- /dev/null +++ b/pytemscript/plugins/serialem_ccd/plugin.py @@ -0,0 +1,368 @@ +# The Leginon software is Copyright 2003 +# The Scripps Research Institute, La Jolla, CA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# The code below is a modified version of: https://github.com/leginon-org/leginon/blob/myami-python3/pyscope/gatansocket.py +# This plugin needs the SERIALEMCCD plugin to be installed in DigitalMicrograph +# The instructions can be found at: https://bio3d.colorado.edu/SerialEM/hlp/html/setting_up_serialem.htm +# +# On the computer running DM, define environment variables: +# SERIALEMCCD_PORT=48890 +# [optional] SERIALEMCCD_DEBUG=1 + +import logging +from typing import Tuple +import numpy as np + +from pytemscript.plugins.serialem_ccd.utils import Message, logwrap, enum_gs +from pytemscript.plugins.serialem_ccd.base_socket import BaseSocket + + +class GatanSEMPlugin(BaseSocket): + """ Main class that uses SerialEMCCD plugin of Digital Micrograph on Gatan PC. + It creates a socket connection to that plugin. + """ + def GetDMVersion(self) -> int: + return self.get_long('GS_GetDMVersion') + + def GetNumberOfCameras(self) -> int: + return self.get_long('GS_GetNumberOfCameras') + + def GetCameraName(self, camera_id: int) -> str: + result = self.ExecuteCameraObjectFunction("CM_GetCameraName", camera_id) + return result.array['longarray'].tobytes().decode('utf-8') + + def GetPluginVersion(self) -> int: + return self.get_long('GS_GetPluginVersion') + + def IsCameraInserted(self, camera_id: int) -> bool: + funcCode = enum_gs['GS_IsCameraInserted'] + message_send = Message(longargs=(funcCode, camera_id)) + message_recv = Message(longargs=(0,), boolargs=(0,)) + self.exchange_messages(message_send, message_recv) + result = bool(message_recv.array['boolargs'][0]) + return result + + def InsertCamera(self, camera_id: int, state: int): + funcCode = enum_gs['GS_InsertCamera'] + message_send = Message(longargs=(funcCode, camera_id), boolargs=(state,)) + message_recv = Message(longargs=(0,)) + self.exchange_messages(message_send, message_recv) + + def SetReadMode(self, mode: int, scaling: float = 1.0): + funcCode = enum_gs['GS_SetReadMode'] + message_send = Message(longargs=(funcCode, mode), dblargs=(scaling,)) + message_recv = Message(longargs=(0,)) + self.exchange_messages(message_send, message_recv) + + def SetShutterNormallyClosed(self, camera_id: int, shutter: int): + funcCode = enum_gs['GS_SetShutterNormallyClosed'] + message_send = Message(longargs=(funcCode, camera_id, shutter)) + message_recv = Message(longargs=(0,)) + self.exchange_messages(message_send, message_recv) + + @logwrap + def SetK2Parameters(self, + readMode: int, + scaling: float = 1.0, + hardwareProc: int = 1, + doseFrac: bool = False, + frameTime: float = 0.25, + alignFrames: bool = False, + saveFrames: bool = False, + filt: str = '', + useCds: bool = False): + funcCode = enum_gs['GS_SetK2Parameters2'] + # rotation and flip for non-frame saving image. It is the same definition + # as in SetFileSaving2. If set to 0, it takes the value from GMS + rotationFlip = 0 + self.save_frames = saveFrames + + flags = 0 + flags += int(useCds) * 2 ** 6 + # settings of unused flags + # anti_alias + reducedSizes = 0 + fullSizes = 0 + + # filter name + filt_str = filt + '\0' + extra = len(filt_str) % 4 + if extra: + npad = 4 - extra + filt_str = filt_str + npad * '\0' + longarray = np.frombuffer(bytes(filt_str, 'utf-8'), dtype=np.int32) + + longs = [ + funcCode, + readMode, + hardwareProc, + rotationFlip, + flags, + ] + bools = [ + doseFrac, + alignFrames, + saveFrames, + ] + doubles = [ + scaling, + frameTime, + reducedSizes, + fullSizes, + 0.0, # dummy3 + 0.0, # dummy4 + ] + + message_send = Message(longargs=longs, boolargs=bools, dblargs=doubles, longarray=longarray) + message_recv = Message(longargs=(0,)) # just return code + self.exchange_messages(message_send, message_recv) + + def setNumGrabSum(self, earlyReturnFrameCount: int, earlyReturnRamGrabs: int): + # pack RamGrabs and earlyReturnFrameCount in one double + self.num_grab_sum = 2**16 * earlyReturnRamGrabs + earlyReturnFrameCount + + def getNumGrabSum(self) -> int: + return self.num_grab_sum + + @logwrap + def SetupFileSaving(self, + rotationFlip, + dirname: str, + rootname: str, + filePerImage: int, + doEarlyReturn: bool = False, + earlyReturnFrameCount: int = 0, + earlyReturnRamGrabs: int = 0, + lzwtiff: bool = False) -> None: + pixelSize = 1.0 + self.setNumGrabSum(earlyReturnFrameCount, earlyReturnRamGrabs) + if self.save_frames and (doEarlyReturn or lzwtiff): + # early return flag + flag = 128 * int(doEarlyReturn) + 8 * int(lzwtiff) + numGrabSum = self.getNumGrabSum() + # set values to pass + longs = [enum_gs['GS_SetupFileSaving2'], rotationFlip, flag, ] + dbls = [pixelSize, numGrabSum, 0., 0., 0., ] + else: + longs = [enum_gs['GS_SetupFileSaving'], rotationFlip, ] + dbls = [pixelSize, ] + bools = [filePerImage, ] + names_str = dirname + '\0' + rootname + '\0' + extra = len(names_str) % 4 + if extra: + npad = 4 - extra + names_str = names_str + npad * '\0' + longarray = np.frombuffer(bytes(names_str, 'utf-8'), dtype=np.int32) + message_send = Message(longargs=longs, boolargs=bools, dblargs=dbls, longarray=longarray) + message_recv = Message(longargs=(0, 0)) + self.exchange_messages(message_send, message_recv) + + def GetFileSaveResult(self) -> Tuple: + #longs = [enum_gs['GS_GetFileSaveResult'], rotationFlip] + message_send = Message(longargs=(enum_gs['GS_GetFileSaveResult'],))#, boolargs=bools, dblargs=dbls, longarray=longarray) + message_recv = Message(longargs=(0, 0, 0)) + self.exchange_messages(message_send, message_recv) + args = message_recv.array['longargs'] + numsaved = args[1] + error = args[2] + return numsaved, error + + def SelectCamera(self, camera_id: int): + funcCode = enum_gs['GS_SelectCamera'] + message_send = Message(longargs=(funcCode, camera_id)) + message_recv = Message(longargs=(0,)) + self.exchange_messages(message_send, message_recv) + + def UpdateK2HardwareDarkReference(self, camera_id: int) -> int: + function_name = 'K2_updateHardwareDarkReference' + return self.ExecuteSendCameraObjectionFunction(function_name, camera_id) + + def PrepareDarkReference(self, camera_id: int) -> int: + function_name = 'CM_PrepareDarkReference' + return self.ExecuteSendCameraObjectionFunction(function_name, camera_id) + + def GetEnergyFilter(self) -> float: + if 'GetEnergyFilter' not in self.filter_functions: + return -1.0 + script = 'if ( %s() ) { Exit(1.0); } else { Exit(-1.0); }' % (self.filter_functions['GetEnergyFilter'],) + return self.ExecuteGetDoubleScript(script) + + def SetEnergyFilter(self, value: bool = False) -> int: + if 'SetEnergyFilter' not in self.filter_functions: + return -1 + if value: + i = 1 + else: + i = 0 + script = '%s(%d); %s' % (self.filter_functions['SetEnergyFilter'], i, self.wait_for_filter) + return self.ExecuteSendScript(script) + + def GetEnergyFilterWidthMax(self) -> float: + if 'GetEnergyFilterWidthMax' not in self.filter_functions: + return -1.0 + script = 'Exit(%s())' % (self.filter_functions['GetEnergyFilterWidthMax'],) + return self.ExecuteGetDoubleScript(script) + + def GetEnergyFilterWidth(self) -> float: + if 'GetEnergyFilterWidth' not in self.filter_functions: + return -1.0 + script = 'Exit(%s())' % (self.filter_functions['GetEnergyFilterWidth'],) + return self.ExecuteGetDoubleScript(script) + + def SetEnergyFilterWidth(self, value: float) -> int: + if 'SetEnergyFilterWidth' not in self.filter_functions: + return -1 + script = 'if ( %s(%f) ) { Exit(1.0); } else { Exit(-1.0); }' % ( + self.filter_functions['SetEnergyFilterWidth'], value) + return self.ExecuteSendScript(script) + + def GetEnergyFilterOffset(self) -> float: + if 'GetEnergyFilterOffset' not in self.filter_functions: + return 0.0 + script = 'Exit(%s())' % (self.filter_functions['GetEnergyFilterOffset'],) + return self.ExecuteGetDoubleScript(script) + + def SetEnergyFilterOffset(self, value: float) -> float: + """ + wjr changing this to use the Gatan function IFSetEnergyOffset, which needs a technique and a value + GMS 3.32,function apparently added in GMS 3.2. Later versions will need to be checked + technique 0: instrument (not available) + technique 1: prism offset (confusing because -10 is 10) + technique 2: HT offset (has no effect on BioQuantum, but HT offset in DM will change) + technique 3: drift tube ( this seems best since the value is consistent with direction) + technique 4: prism adjust (confusing because -10 is 10, and it does not count when checking the energy loss value) + note: the Gatan function being called is a void, so removed the boolean logic used for most other functions + """ + technique = 3 # hard code to drift tube for now + if 'SetEnergyFilterOffset' not in self.filter_functions: + return -1.0 + script = '%s(%i,%f)' % (self.filter_functions['SetEnergyFilterOffset'], technique, value) + self.ExecuteSendScript(script) + # return 1 + # or better to + newvalue = self.GetEnergyFilterOffset() # ? but wastes time + if value == newvalue: + return 1 + else: + technique = 2 # reset the HT offset to 0, sometimes this gets set + script = '%s(%i,%f)' % (self.filter_functions['SetEnergyFilterOffset'], technique, 0) + self.ExecuteSendScript(script) + newvalue = self.GetEnergyFilterOffset() + if value == newvalue: + return 1 + else: + return -1 + + def AlignEnergyFilterZeroLossPeak(self) -> float: + script = ' if ( %s() ) { %s Exit(1.0); } else { Exit(-1.0); }' % ( + self.filter_functions['AlignEnergyFilterZeroLossPeak'], self.wait_for_filter) + return self.ExecuteGetDoubleScript(script) + + @logwrap + def GetImage(self, + processing: str, + height: int, + width: int, + binning: int, + top: int, + left: int, + bottom: int, + right: int, + exposure: float, + corrections: int, + shutter: int = 0, + shutterDelay: float = 0.0) -> np.ndarray: + + arrSize = width * height + + # TODO: need to figure out what these should be + divideBy2 = 0 + settling = 0.0 + + # prepare args for message + if processing == 'dark': + longargs = [enum_gs['GS_GetDarkReference']] + else: + longargs = [enum_gs['GS_GetAcquiredImage']] + longargs.extend([ + arrSize, # pixels in the image + width, height, + ]) + if processing == 'unprocessed': + longargs.append(0) + elif processing == 'dark subtracted': + longargs.append(1) + elif processing == 'gain normalized': + longargs.append(2) + longargs.extend([ + binning, + top, left, bottom, right, + shutter, + ]) + if processing != 'dark': + longargs.append(shutterDelay) + longargs.extend([ + divideBy2, + corrections, + ]) + dblargs = [ + exposure, + settling, + ] + + message_send = Message(longargs=longargs, dblargs=dblargs) + message_recv = Message(longargs=(0, 0, 0, 0, 0)) + self.exchange_messages(message_send, message_recv) + + longargs = message_recv.array['longargs'].tolist() + logging.debug('GetImage longargs %s', longargs) + if longargs[0] < 0: + return 1 # FIXME + arrSize = longargs[1] + width = longargs[2] + height = longargs[3] + numChunks = longargs[4] + bytesPerPixel = 2 # depends on the results formated =uint16 + numBytes = arrSize * bytesPerPixel + chunkSize = (numBytes + numChunks - 1) // numChunks + imArray = np.zeros((height * width,), np.uint16) + received = 0 + remain = numBytes + index = 0 + logging.debug('chunk size %d', chunkSize) + for chunk in range(numChunks): + recv_bytes = b'' + # send chunk handshake for all but the first chunk + if chunk: + message_send = Message(longargs=(enum_gs['GS_ChunkHandshake'],)) + self.exchange_messages(message_send) + thisChunkSize = min(remain, chunkSize) + chunkReceived = 0 + chunkRemain = thisChunkSize + while chunkRemain: + new_recv = self.recv_data(chunkRemain) + len_recv = len(new_recv) + recv_bytes += new_recv + chunkReceived += len_recv + chunkRemain -= len_recv + remain -= len_recv + received += len_recv + last_index = int(index) + logging.debug('chunk bytes length %d', len(recv_bytes)) + index = chunkSize * (chunk + 1) // bytesPerPixel + logging.debug('array index range to fill %d:%d', last_index, index) + imArray[last_index:index] = np.frombuffer(recv_bytes, dtype=np.uint16) + imArray = imArray.reshape((height, width)) + return imArray diff --git a/pytemscript/plugins/serialem_ccd/utils.py b/pytemscript/plugins/serialem_ccd/utils.py new file mode 100644 index 0000000..7973e4f --- /dev/null +++ b/pytemscript/plugins/serialem_ccd/utils.py @@ -0,0 +1,128 @@ +# The Leginon software is Copyright 2003 +# The Scripps Research Institute, La Jolla, CA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# The code below is a modified version of: https://github.com/leginon-org/leginon/blob/myami-python3/pyscope/gatansocket.py + +import functools +import numpy as np +import logging + +# enum function codes as in https://github.com/mastcu/SerialEMCCD/blob/master/SocketPathway.cpp +# need to match exactly both in number and order +enum_gs = [ + 'GS_ExecuteScript', + 'GS_SetDebugMode', + 'GS_SetDMVersion', + 'GS_SetCurrentCamera', + 'GS_QueueScript', + 'GS_GetAcquiredImage', + 'GS_GetDarkReference', + 'GS_GetGainReference', + 'GS_SelectCamera', + 'GS_SetReadMode', + 'GS_GetNumberOfCameras', + 'GS_IsCameraInserted', + 'GS_InsertCamera', + 'GS_GetDMVersion', + 'GS_GetDMCapabilities', + 'GS_SetShutterNormallyClosed', + 'GS_SetNoDMSettling', + 'GS_GetDSProperties', + 'GS_AcquireDSImage', + 'GS_ReturnDSChannel', + 'GS_StopDSAcquisition', + 'GS_CheckReferenceTime', + 'GS_SetK2Parameters', + 'GS_ChunkHandshake', # deleted in the new plugin? + 'GS_SetupFileSaving', + 'GS_GetFileSaveResult', + 'GS_SetupFileSaving2', + 'GS_GetDefectList', + 'GS_SetK2Parameters2', + 'GS_StopContinuousCamera', + 'GS_GetPluginVersion', + 'GS_GetLastError', + 'GS_FreeK2GainReference', + 'GS_IsGpuAvailable', + 'GS_SetupFrameAligning', + 'GS_FrameAlignResults', + 'GS_ReturnDeferredSum', + 'GS_MakeAlignComFile', + 'GS_WaitUntilReady', + 'GS_GetLastDoseRate', + 'GS_SaveFrameMdoc', + 'GS_GetDMVersionAndBuild', + 'GS_GetTiltSumProperties', +] +# lookup table of function name to function code, starting with 1 +enum_gs = {item: index for index, item in enumerate(enum_gs, 1)} + +## C "long" -> numpy "int32" +ARGS_BUFFER_SIZE = 1024 +MAX_LONG_ARGS = 16 +MAX_DBL_ARGS = 8 +MAX_BOOL_ARGS = 8 +sArgsBuffer = np.zeros(ARGS_BUFFER_SIZE, dtype=np.byte) + + +class Message: + """ Information packet to send and receive on the socket. """ + def __init__(self, + longargs=(), boolargs=(), dblargs=(), + longarray=np.array([], dtype=np.int32)): + """ Initialize with the sequences of args (longs, bools, doubles) + and optional long array. """ + + if len(longarray): + longargs = (*longargs, len(longarray)) + + self.dtype = [ + ('size', np.intc, (1,)), + ('longargs', np.int32, (len(longargs),)), + ('boolargs', np.int32, (len(boolargs),)), + ('dblargs', np.double, (len(dblargs),)), + ('longarray', np.int32, (len(longarray),)), + ] + self.array = np.empty((), dtype=self.dtype) + self.array['size'] = np.array([self.array.nbytes], dtype=np.intc) + self.array['longargs'] = np.array(longargs, dtype=np.int32) + self.array['boolargs'] = np.array(boolargs, dtype=np.int32) + self.array['dblargs'] = np.array(dblargs, dtype=np.double) + self.array['longarray'] = longarray + + def pack(self) -> memoryview: + """ Serialize the data. """ + packed = self.array.nbytes + if packed > ARGS_BUFFER_SIZE: + raise RuntimeError('Message packet size %d is larger than maximum %d' % (packed, ARGS_BUFFER_SIZE)) + return self.array.data + + def unpack(self, buf: bytes) -> None: + """ Unpack buffer into our data structure. """ + self.array = np.frombuffer(buf, dtype=self.dtype)[0] + + +def logwrap(func): + """ Decorator to log socket send and recv calls. """ + @functools.wraps(func) + def wrapper(*args, **kwargs): + logging.debug('%s\t%r\t%r', func.__name__, args, kwargs) + try: + return func(*args, **kwargs) + except Exception as e: + logging.error('EXCEPTION: %s', e) + raise + + return wrapper diff --git a/tests/test_sem_ccd.py b/tests/test_sem_ccd.py new file mode 100644 index 0000000..e3991cc --- /dev/null +++ b/tests/test_sem_ccd.py @@ -0,0 +1,94 @@ +from pytemscript.plugins.serialem_ccd.plugin import GatanSEMPlugin + + +def main(): + g = GatanSEMPlugin(host="192.168.71.2", debug=True) + print('Version', g.GetDMVersion()) + print('GetNumberOfCameras', g.GetNumberOfCameras()) + print('GetPluginVersion', g.GetPluginVersion()) + print("SelectCamera") + g.SelectCamera(0) + print("GetCameraName", g.GetCameraName(0)) + if not g.IsCameraInserted(0): + print('InsertCamera') + g.InsertCamera(0, True) + print('IsCameraInserted', g.IsCameraInserted(0)) + + k2params = { + 'readMode': 1, # {'non-K2 cameras':-1, 'linear': 0, 'counting': 1, 'super resolution': 2} + 'scaling': 1.0, + 'hardwareProc': 1, # {'none': 0, 'dark': 2, 'gain': 4, 'dark+gain': 6} + 'doseFrac': False, + 'frameTime': 0.25, + 'alignFrames': False, + 'saveFrames': False, + 'filt': 'None' + } + + + def getDMVersion(g): + """ Return DM version details: version_long, major.minor.sub """ + version_long = g.GetDMVersion() + if version_long < 40000: + major = 1 + minor = None + sub = None + if version_long >= 31100: + # 2.3.0 gives an odd number of 31100 + major = 2 + minor = 3 + sub = 0 + version_long = 40300 + elif version_long == 40000: + # minor version can be 0 or 1 in this case + # but likely 1 since we have not used this module until k2 is around + major = 2 + minor = 1 + sub = 0 + else: + major = version_long // 10000 - 2 + remainder = version_long - (major + 2) * 10000 + minor = remainder // 100 + sub = remainder % 100 + + return version_long, '%d.%d.%d' % (major, minor, sub) + + + def getCorrectionFlags(g): + """ + Binary Correction flag sum in GMS. See Feature #8391. + GMS 3.3.2 has pre-counting correction which is superior. + SerialEM always does this correction. + David M. said SerialEM default is 49 for K2 and 1 for K3. + 1 means defect correction only. + 49 means defect, bias, and quadrant (to be the same as Ultrascan). + I don't think the latter two need applying in counting. + """ + version_id, version_string = getDMVersion(g) + isDM332orUp = version_id >= 50302 + return 1 if isDM332orUp else 0 + + + acqparams = { + 'processing': 'unprocessed', # dark, dark subtracted, gain normalized + 'height': 2048, + 'width': 2048, + 'binning': 1, + 'top': 0, + 'left': 0, + 'bottom': 2048, + 'right': 2048, + 'exposure': 1.0, + 'corrections': getCorrectionFlags(g), + 'shutter': 0, + 'shutterDelay': 0.0, + } + + # g.SetK2Parameters(**k2params) + g.SetReadMode(-1) + image = g.GetImage(**acqparams) + print("Got image array:", image.dtype, image.shape) + + +if __name__ == '__main__': + main() From 31f24ca2f7f640e6e134f84e4a91be8a3c7c34e1 Mon Sep 17 00:00:00 2001 From: Grigory Sharov Date: Mon, 24 Mar 2025 20:03:43 +0000 Subject: [PATCH 06/11] working on stem multichannel acq --- pytemscript/modules/acquisition.py | 177 +++++++++++++++++++++-------- 1 file changed, 127 insertions(+), 50 deletions(-) diff --git a/pytemscript/modules/acquisition.py b/pytemscript/modules/acquisition.py index 193b0d4..bec06ed 100644 --- a/pytemscript/modules/acquisition.py +++ b/pytemscript/modules/acquisition.py @@ -1,4 +1,4 @@ -from typing import Optional, Dict +from typing import Optional, Dict, List, Union import time import logging from datetime import datetime @@ -109,24 +109,35 @@ def show_cameras_cca(self, tem_cameras: Dict) -> Dict: return tem_cameras - def acquire(self, cameraName: str, **kwargs) -> Image: + def acquire(self, cameraName: Union[str, List], **kwargs) -> Union[Image, List[Image]]: """ Perform actual acquisition. Camera settings should be set beforehand. - :param cameraName: Camera name - :returns: Image object + :param cameraName: Camera name(s) + :returns: Image object or a list of Image objects """ + if isinstance(cameraName, str): + cameraName = [cameraName] + acq = self.com_object acq.RemoveAllAcqDevices() - acq.AddAcqDeviceByName(cameraName) + for c in cameraName: + acq.AddAcqDeviceByName(c) t0 = time.time() imgs = acq.AcquireImages() t1 = time.time() - image = convert_image(imgs[0], name=cameraName, **kwargs) + + result = [] + for idx, c in enumerate(cameraName): + image = convert_image(imgs[idx], name=c, **kwargs) + result.append(image) t2 = time.time() logging.debug("\tAcquisition took %f s" % (t1 - t0)) - logging.debug("\tConverting image took %f s" %(t2 - t1)) + logging.debug("\tConverting image(s) took %f s" %(t2 - t1)) - return image + if len(result) == 1: + return result[0] + else: + return result def acquire_advanced(self, cameraName: str, @@ -326,23 +337,34 @@ def set_tem_presets_advanced(self, total, output) def set_stem_presets(self, - cameraName: str, + cameraName: Union[str, List], size: AcqImageSize = AcqImageSize.FULL, dwell_time: float = 1e-5, binning: int = 1, **kwargs) -> None: + if isinstance(cameraName, str): + cameraName = [cameraName] + + brightness = kwargs.get('brightness') + if isinstance(brightness, str): + brightness = [brightness] + + contrast = kwargs.get('contrast') + if isinstance(contrast, str): + contrast = [contrast] + + for idx, cam in enumerate(cameraName): + for obj in self.com_object: + if obj.Info.Name == cam: + self.current_camera = obj + if brightness is not None: + self.current_camera.Info.Brightness = brightness[idx] + if contrast is not None: + self.current_camera.Info.Contrast = contrast[idx] + break - for stem in self.com_object: - if stem.Info.Name == cameraName: - self.current_camera = stem - break - if self.current_camera is None: - raise KeyError("No STEM detector with name %s" % cameraName) - - if 'brightness' in kwargs: - self.current_camera.Info.Brightness = kwargs['brightness'] - if 'contrast' in kwargs: - self.current_camera.Info.Contrast = kwargs['contrast'] + if self.current_camera is None: + raise KeyError("No STEM detector with name %s" % cam) settings = self.com_object.AcqParams # StemAcqParams settings.ImageSize = size @@ -382,18 +404,23 @@ def __has_film(self) -> bool: return self.__client.call(method="has", body=body) @staticmethod - def __find_camera(cameraName: str, - cameras_dict: Dict, + def __find_camera(cameraName: Union[str, List], + all_cameras: Dict, binning: int) -> Dict: - """ Check camera name and supported binning. """ - camera_dict = cameras_dict.get(cameraName) + """ Check camera name(s) and supported binning. """ + if isinstance(cameraName, str): + cameraName = [cameraName] - if camera_dict is None: - raise KeyError("No camera with name %s. If using standard scripting the " - "camera must be selected in the microscope user interface" % cameraName) + camera_dict = None + for c in cameraName: + camera_dict = all_cameras.get(c) - if binning not in camera_dict["binnings"]: - raise ValueError("Unsupported binning value: %d" % binning) + if camera_dict is None: + raise KeyError("No camera with name %s. If using standard scripting the " + "camera must be selected in the microscope user interface" % cameraName) + + if binning not in camera_dict["binnings"]: + raise ValueError("Unsupported binning value: %d" % binning) return camera_dict @@ -453,15 +480,15 @@ def __acquire_with_tecnaiccd(self, return image def acquire_tem_image(self, - cameraName: str, + camera: str, size: AcqImageSize = AcqImageSize.FULL, exp_time: float = 1.0, binning: int = 1, **kwargs) -> Optional[Image]: """ Acquire a TEM image. - :param cameraName: Camera name - :type cameraName: str + :param camera: Camera name + :type camera: str :param size: Image size (AcqImageSize enum) :type size: IntEnum :param exp_time: Exposure time in seconds @@ -497,10 +524,10 @@ def acquire_tem_image(self, >>> print(img.width) 4096 """ - camera_dict = self.__find_camera(cameraName, self.cameras, binning) + camera_dict = self.__find_camera(camera, self.cameras, binning) if kwargs.get("use_tecnaiccd", False): - return self.__acquire_with_tecnaiccd(cameraName, size, exp_time, + return self.__acquire_with_tecnaiccd(camera, size, exp_time, binning, camera_dict["width"], **kwargs) @@ -513,7 +540,7 @@ def acquire_tem_image(self, body = RequestBody(attr="tem.Acquisition.Cameras", obj_cls=AcquisitionObj, obj_method="set_tem_presets", - cameraName=cameraName, + cameraName=camera, size=size, exp_time=exp_time, binning=binning, @@ -524,16 +551,16 @@ def acquire_tem_image(self, body = RequestBody(attr="tem.Acquisition", obj_cls=AcquisitionObj, obj_method="acquire", - cameraName=cameraName, + cameraName=camera, **kwargs) image = self.__client.call(method="exec_special", body=body) - logging.info("TEM image acquired on %s", cameraName) + logging.info("TEM image acquired on %s", camera) if prev_shutter_mode is not None: body = RequestBody(attr="tem.Acquisition.Cameras", obj_cls=AcquisitionObj, obj_method="restore_shutter", - cameraName=cameraName, + cameraName=camera, prev_shutter_mode=prev_shutter_mode) self.__client.call(method="exec_special", body=body) @@ -543,7 +570,7 @@ def acquire_tem_image(self, body = RequestBody(attr=self.__id_adv, obj_cls=AcquisitionObj, obj_method="set_tem_presets_advanced", - cameraName=cameraName, + cameraName=camera, size=size, exp_time=exp_time, binning=binning, @@ -555,32 +582,32 @@ def acquire_tem_image(self, body = RequestBody(attr=self.__id_adv, obj_cls=AcquisitionObj, obj_method="acquire_advanced", - cameraName=cameraName, + cameraName=camera, recording=kwargs["recording"], **kwargs) self.__client.call(method="exec_special", body=body) - logging.info("TEM image acquired on %s", cameraName) + logging.info("TEM image acquired on %s", camera) return None else: body = RequestBody(attr=self.__id_adv, validator=Image, obj_cls=AcquisitionObj, obj_method="acquire_advanced", - cameraName=cameraName, + cameraName=camera, **kwargs) image = self.__client.call(method="exec_special", body=body) return image def acquire_stem_image(self, - cameraName: str, + detector: str, size: AcqImageSize = AcqImageSize.FULL, dwell_time: float = 1e-5, binning: int = 1, **kwargs) -> Image: - """ Acquire a STEM image. + """ Acquire a single STEM image. - :param cameraName: Camera name - :type cameraName: str + :param detector: Detector name + :type detector: str :param size: Image size (AcqImageSize enum) :type size: IntEnum :param dwell_time: Dwell time in seconds. The frame time equals the dwell time times the number of pixels plus some overhead (typically 20%, used for the line flyback) @@ -591,12 +618,12 @@ def acquire_stem_image(self, :keyword float contrast: Contrast setting (0.0-1.0) :returns: Image object """ - _ = self.__find_camera(cameraName, self.stem_detectors, binning) + _ = self.__find_camera(detector, self.stem_detectors, binning) body = RequestBody(attr="tem.Acquisition.Detectors", obj_cls=AcquisitionObj, obj_method="set_stem_presets", - cameraName=cameraName, + cameraName=detector, size=size, dwell_time=dwell_time, binning=binning, @@ -608,13 +635,63 @@ def acquire_stem_image(self, validator=Image, obj_cls=AcquisitionObj, obj_method="acquire", - cameraName=cameraName, + cameraName=detector, **kwargs) image = self.__client.call(method="exec_special", body=body) - logging.info("STEM image acquired on %s", cameraName) + logging.info("STEM image acquired on %s", detector) return image + def acquire_stem_images(self, + detectors: List[str], + size: AcqImageSize, + dwell_time: float = 1e-5, + binning: int = 1, + **kwargs) -> List[Image]: + """ Simultaneous acquisition of multiple STEM images. + + :param detectors: List of STEM detector names + :type detectors: list + :param size: Image size (AcqImageSize enum) + :type size: IntEnum + :param dwell_time: Dwell time in seconds. The frame time equals the dwell time times the number of pixels plus some overhead (typically 20%, used for the line flyback) + :type dwell_time: float + :param binning: Binning factor. Technically speaking these are "pixel skipping" values, since in STEM we do not combine pixels as a CCD does. + :type binning: int + :keyword list brightness: list of Brightness settings for each detector + :keyword list contrast: list of Contrast settings for each detector + :returns: list of Image objects + """ + if "brightness" in kwargs: + assert len(kwargs["brightness"]) == len(detectors) + if "contrast" in kwargs: + assert len(kwargs["contrast"]) == len(detectors) + + _ = self.__find_camera(detectors, self.stem_detectors, binning) + + body = RequestBody(attr="tem.Acquisition.Detectors", + obj_cls=AcquisitionObj, + obj_method="set_stem_presets", + cameraName=detectors, + size=size, + dwell_time=dwell_time, + binning=binning, + **kwargs) + self.__client.call(method="exec_special", body=body) + + self.__check_prerequisites() + body = RequestBody(attr="tem.Acquisition", + validator=Image, + obj_cls=AcquisitionObj, + obj_method="acquire", + cameraName=detectors, + **kwargs) + images = self.__client.call(method="exec_special", body=body) + logging.info("STEM images acquired on %s", detectors) + + return images + + def acquire_film(self, film_text: str, exp_time: float) -> None: From 4be1a86c36407b38bd6f8a53dc9a1643d4b4a07e Mon Sep 17 00:00:00 2001 From: Grigory Sharov Date: Wed, 26 Mar 2025 15:18:06 +0000 Subject: [PATCH 07/11] expand piezo methods --- pytemscript/modules/extras.py | 26 ++++++- pytemscript/modules/piezo_stage.py | 113 +++++++++++++++++++++++++++-- pytemscript/modules/stage.py | 1 - 3 files changed, 132 insertions(+), 8 deletions(-) diff --git a/pytemscript/modules/extras.py b/pytemscript/modules/extras.py index bd0420c..1418a56 100644 --- a/pytemscript/modules/extras.py +++ b/pytemscript/modules/extras.py @@ -248,12 +248,13 @@ def set(self, axes: int = 0, speed: Optional[float] = None, method: str = "MoveTo", + piezo: bool = False, **kwargs) -> None: """ Execute stage move to a new position. """ if method not in ["MoveTo", "GoTo", "GoToWithSpeed"]: raise NotImplementedError("Method %s is not implemented" % method) - pos = self.com_object.Position + pos = self.com_object.CurrentPosition if piezo else self.com_object.Position for key, value in kwargs.items(): setattr(pos, key.upper(), float(value)) @@ -262,13 +263,22 @@ def set(self, else: getattr(self.com_object, method)(pos, axes) + def start_jog(self, axes: int = 0, **kwargs) -> None: + """ Start jogging with specified velocities. """ + speed = self.com_object.CurrentJogVelocity + + for key, value in kwargs.items(): + setattr(speed, key.upper(), float(value)) + + getattr(self.com_object, "StartJog")(speed, axes) + def get(self, a=False, b=False) -> Dict: """ The current position of the stage/piezo stage (x,y,z in um). Set a and b to True if you want to retrieve them as well. x,y,z are in um and a,b in deg If retrieving velocity, return the speed of the piezo stage instead. - x,y,z are in um/s and a,b in deg/s + x,y,z are in um/s """ pos = OrderedDict((key, getattr(self.com_object, key.upper()) * 1e6) for key in 'xyz') if a: @@ -291,3 +301,15 @@ def limits(self) -> Dict: } return limits + + def limits_piezo(self) -> Dict: + """ Returns a dict with stage move limits for piezo stage. """ + limits = OrderedDict() + min_pos, max_pos = self.com_object.GetPositionRange + for axis in 'xyz': + limits[axis] = { + 'min': getattr(min_pos, axis.upper()), + 'max': getattr(max_pos, axis.upper()) + } + + return limits diff --git a/pytemscript/modules/piezo_stage.py b/pytemscript/modules/piezo_stage.py index f66e9a3..444c514 100644 --- a/pytemscript/modules/piezo_stage.py +++ b/pytemscript/modules/piezo_stage.py @@ -2,6 +2,7 @@ from typing import Dict, Tuple from ..utils.misc import RequestBody +from ..utils.enums import StageAxes from .extras import StageObj @@ -23,13 +24,13 @@ def __has_pstage(self) -> bool: @property def position(self) -> Dict: - """ The current position of the piezo stage (x,y,z in um and a,b in degrees). """ + """ The current position of the piezo stage (x,y,z in um). """ if not self.__has_pstage: raise NotImplementedError(self.__err_msg) else: body = RequestBody(attr=self.__id + ".CurrentPosition", validator=dict, - obj_cls=StageObj, obj_method="get", a=True) + obj_cls=StageObj, obj_method="get") return self.__client.call(method="exec_special", body=body) @property @@ -43,12 +44,114 @@ def position_range(self) -> Tuple[float, float]: @property def velocity(self) -> Dict: - """ Returns a dict with stage velocities (x,y,z are in um/s and a,b in deg/s). """ + """ Returns a dict with current jogging velocities (x,y,z are in um/s). """ if not self.__has_pstage: raise NotImplementedError(self.__err_msg) else: body = RequestBody(attr=self.__id + ".CurrentJogVelocity", validator=dict, - obj_cls=StageObj, obj_method="get", - get_speed=True) + obj_cls=StageObj, obj_method="get") + return self.__client.call(method="exec_special", body=body) + + @property + def high_resolution(self) -> bool: + """ """ + if not self.__has_pstage: + raise NotImplementedError(self.__err_msg) + else: + body = RequestBody(attr=self.__id + ".HighResolution", + validator=bool) + return self.__client.call(method="get", body=body) + + @high_resolution.setter + def high_resolution(self, value: bool) -> None: + if not self.__has_pstage: + raise NotImplementedError(self.__err_msg) + else: + body = RequestBody(attr=self.__id + ".HighResolution", value=value) + self.__client.call(method="set", body=body) + + def go_to(self, **kwargs) -> None: + """ Move piezo stage to the new position. + Keyword args can be x,y,z (in um) + """ + if not self.__has_pstage: + raise NotImplementedError(self.__err_msg) + else: + + # convert units to meters and radians + new_coords = dict() + for axis in 'xyz': + if kwargs.get(axis) is not None: + new_coords.update({axis: kwargs[axis] * 1e-6}) + + limits = self.limits + axes = 0 + for key, value in new_coords.items(): + if key not in 'xyz': + raise ValueError("Unexpected axis: %s" % key) + if value < limits[key]['min'] or value > limits[key]['max']: + raise ValueError('Stage position %s=%s is out of range' % (value, key)) + axes |= getattr(StageAxes, key.upper()) + + body = RequestBody(attr=self.__id, obj_cls=StageObj, + obj_method="set", axes=axes, + method="GoTo", piezo=True, **new_coords) + self.__client.call(method="exec_special", body=body) + + def start_jogging(self, **kwargs) -> None: + """ Start jogging with specified velocities for each axis. + Keyword args can be x,y,z (in um/s) + """ + if not self.__has_pstage: + raise NotImplementedError(self.__err_msg) + else: + # convert units to meters + new_speed = dict() + for axis in 'xyz': + if kwargs.get(axis) is not None: + new_speed.update({axis: kwargs[axis] * 1e-6}) + + axes = 0 + for key, value in new_speed.items(): + if key not in 'xyz': + raise ValueError("Unexpected axis: %s" % key) + axes |= getattr(StageAxes, key.upper()) + + body = RequestBody(attr=self.__id, obj_cls=StageObj, + obj_method="start_jog", axes=axes, + **new_speed) + self.__client.call(method="exec_special", body=body) + + def stop_jogging(self, axis: StageAxes) -> None: + """ Stop jogging for specified axis. + :param axis: axis to stop jogging (StageAxes enum) + :type axis: StageAxes + """ + if not self.__has_pstage: + raise NotImplementedError(self.__err_msg) + else: + body = RequestBody(attr=self.__id + ".StopJog()", arg=axis) + self.__client.call(method="exec", body=body) + + def reset_position(self, axis: StageAxes) -> None: + """ Reset position for specified axis. + :param axis: axis to reset (StageAxes enum) + :type axis: StageAxes + """ + if not self.__has_pstage: + raise NotImplementedError(self.__err_msg) + else: + body = RequestBody(attr=self.__id + ".ResetPosition()", arg=axis) + self.__client.call(method="exec", body=body) + + @property + @lru_cache(maxsize=1) + def limits(self) -> Dict: + """ Returns a dict with piezo stage move limits. """ + if not self.__has_pstage: + raise NotImplementedError(self.__err_msg) + else: + body = RequestBody(attr=self.__id, validator=dict, + obj_cls=StageObj, obj_method="limits_piezo") return self.__client.call(method="exec_special", body=body) diff --git a/pytemscript/modules/stage.py b/pytemscript/modules/stage.py index 8a6a713..191fcec 100644 --- a/pytemscript/modules/stage.py +++ b/pytemscript/modules/stage.py @@ -58,7 +58,6 @@ def _change_position(self, new_coords = dict() for axis in 'xyz': if kwargs.get(axis) is not None: - new_coords.update({axis: kwargs[axis] * 1e-6}) for axis in 'ab': if kwargs.get(axis) is not None: From 74978e02aa908e024c7942b9e68700eab6b4662b Mon Sep 17 00:00:00 2001 From: Grigory Sharov Date: Wed, 26 Mar 2025 15:21:35 +0000 Subject: [PATCH 08/11] update changelog --- docs/changelog.rst | 8 ++++++++ docs/conf.py | 2 +- pytemscript/__init__.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1f7120e..0c14aa5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,6 +1,14 @@ Changelog ========= +Version 3.1 +^^^^^^^^^^^ + +* Simultaneous STEM acquisition for multiple detectors +* Use CalGetter to get magnifications and camera lengths +* Add more methods for piezo stage (untested) +* SerialEMCCD plugin for advanced Gatan cameras + Version 3.0 ^^^^^^^^^^^ diff --git a/docs/conf.py b/docs/conf.py index ea1f0b6..af9b75e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -21,7 +21,7 @@ author = 'Tore Niermann, Grigory Sharov' # The full version, including alpha/beta/rc tags -release = '3.0b3' +release = '3.1' # -- General configuration --------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. diff --git a/pytemscript/__init__.py b/pytemscript/__init__.py index 6feaacf..27d4bb6 100644 --- a/pytemscript/__init__.py +++ b/pytemscript/__init__.py @@ -1 +1 @@ -__version__ = '3.0b3' +__version__ = '3.1' From 25f06dda54f5654b52946ec7e6ed995399dfbb88 Mon Sep 17 00:00:00 2001 From: Grigory Sharov Date: Fri, 4 Apr 2025 11:55:23 +0100 Subject: [PATCH 09/11] minor changes --- .github/workflows/publish_and_tag.yml | 8 ++------ pyproject.toml.future | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/publish_and_tag.yml b/.github/workflows/publish_and_tag.yml index 072a087..f5e0ec7 100644 --- a/.github/workflows/publish_and_tag.yml +++ b/.github/workflows/publish_and_tag.yml @@ -1,13 +1,9 @@ -# Workflow to send master to pypi and tag the branch -name: master to pypi with comments and tag - +# Workflow to send master to pypi +name: master to pypi # Triggers the workflow on push to the master branch on: push: branches: [ master ] - -env: - FOLDER_WITH_VERSION: pytemscript # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: deploy: diff --git a/pyproject.toml.future b/pyproject.toml.future index 403b175..8b170e5 100644 --- a/pyproject.toml.future +++ b/pyproject.toml.future @@ -12,7 +12,7 @@ authors = [ description = "TEM Scripting adapter for FEI/TFS microscopes" readme = {file = "README.rst", content-type = "text/x-rst"} requires-python = ">=3.8" -keywords = ["TEM python"] +keywords = ["TEM", "python"] classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Science/Research', From b33fe6c30aef0de3e5226edda224700c6723834d Mon Sep 17 00:00:00 2001 From: Grigory Sharov Date: Fri, 4 Apr 2025 11:55:35 +0100 Subject: [PATCH 10/11] check avail axes for piezo --- pytemscript/modules/extras.py | 4 +++- pytemscript/modules/piezo_stage.py | 22 +++++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/pytemscript/modules/extras.py b/pytemscript/modules/extras.py index 2c34bdf..e92ec1a 100644 --- a/pytemscript/modules/extras.py +++ b/pytemscript/modules/extras.py @@ -324,8 +324,10 @@ def limits(self) -> Dict: def limits_piezo(self) -> Dict: """ Returns a dict with stage move limits for piezo stage. """ limits = OrderedDict() + axes = self.com_object.AvailableAxes + avail_axes = [member.name for member in StageAxes if axes & member.value] min_pos, max_pos = self.com_object.GetPositionRange - for axis in 'xyz': + for axis in avail_axes: limits[axis] = { 'min': getattr(min_pos, axis.upper()), 'max': getattr(max_pos, axis.upper()) diff --git a/pytemscript/modules/piezo_stage.py b/pytemscript/modules/piezo_stage.py index 444c514..0de6b49 100644 --- a/pytemscript/modules/piezo_stage.py +++ b/pytemscript/modules/piezo_stage.py @@ -1,5 +1,5 @@ from functools import lru_cache -from typing import Dict, Tuple +from typing import Dict from ..utils.misc import RequestBody from ..utils.enums import StageAxes @@ -33,15 +33,6 @@ def position(self) -> Dict: obj_cls=StageObj, obj_method="get") return self.__client.call(method="exec_special", body=body) - @property - def position_range(self) -> Tuple[float, float]: - """ Return min and max positions. """ - if not self.__has_pstage: - raise NotImplementedError(self.__err_msg) - else: - body = RequestBody(attr=self.__id + ".GetPositionRange()") - return self.__client.call(method="exec", body=body) - @property def velocity(self) -> Dict: """ Returns a dict with current jogging velocities (x,y,z are in um/s). """ @@ -78,7 +69,7 @@ def go_to(self, **kwargs) -> None: if not self.__has_pstage: raise NotImplementedError(self.__err_msg) else: - + self.__check_limits(**kwargs) # convert units to meters and radians new_coords = dict() for axis in 'xyz': @@ -106,6 +97,7 @@ def start_jogging(self, **kwargs) -> None: if not self.__has_pstage: raise NotImplementedError(self.__err_msg) else: + self.__check_limits(**kwargs) # convert units to meters new_speed = dict() for axis in 'xyz': @@ -155,3 +147,11 @@ def limits(self) -> Dict: body = RequestBody(attr=self.__id, validator=dict, obj_cls=StageObj, obj_method="limits_piezo") return self.__client.call(method="exec_special", body=body) + + def __check_limits(self, **kwargs) -> None: + """ Check if input axes are available. """ + available_axes = self.limits.keys() + input_axes = kwargs.keys() + missing_axes = set(input_axes - available_axes) + if missing_axes: + raise ValueError("Available piezo axes are: %s" % available_axes) From 69171a69670567e07a2b85a8ab932d6176e2ab42 Mon Sep 17 00:00:00 2001 From: Grigory Sharov Date: Fri, 4 Apr 2025 12:29:41 +0100 Subject: [PATCH 11/11] fixes for multi-detector STEM acq --- pytemscript/modules/acquisition.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/pytemscript/modules/acquisition.py b/pytemscript/modules/acquisition.py index 6ff2040..7ce5813 100644 --- a/pytemscript/modules/acquisition.py +++ b/pytemscript/modules/acquisition.py @@ -109,7 +109,9 @@ def show_cameras_cca(self, tem_cameras: Dict) -> Dict: return tem_cameras - def acquire(self, cameraName: Union[str, List], **kwargs) -> Union[Image, List[Image]]: + def acquire(self, + cameraName: Union[str, List], + **kwargs) -> Union[Image, List[Image]]: """ Perform actual acquisition. Camera settings should be set beforehand. :param cameraName: Camera name(s) @@ -418,10 +420,11 @@ def __find_camera(cameraName: Union[str, List], if camera_dict is None: raise KeyError("No camera with name %s. If using standard scripting the " - "camera must be selected in the microscope user interface" % cameraName) + "camera must be selected in the microscope user interface" % c) if binning not in camera_dict["binnings"]: - raise ValueError("Unsupported binning value: %d" % binning) + raise ValueError("Unsupported binning value: %d for camera %s" % ( + binning, c)) return camera_dict @@ -640,25 +643,25 @@ def acquire_stem_image(self, def acquire_stem_images(self, detectors: List[str], - size: AcqImageSize, + size: AcqImageSize = AcqImageSize.FULL, dwell_time: float = 1e-5, binning: int = 1, **kwargs) -> List[Image]: """ Simultaneous acquisition of multiple STEM images. - :param list detectors: List of STEM detector names + :param List[str] detectors: List of STEM detector names :param AcqImageSize size: Image size :param float dwell_time: Dwell time in seconds. The frame time equals the dwell time times the number of pixels plus some overhead (typically 20%, used for the line flyback) :param int binning: Binning factor. Technically speaking these are "pixel skipping" values, since in STEM we do not combine pixels as a CCD does. - :keyword list brightness: list of Brightness settings for each detector - :keyword list contrast: list of Contrast settings for each detector + :keyword List[float] brightness: list of Brightness settings for each detector + :keyword List[float] contrast: list of Contrast settings for each detector :returns: list of Image objects :rtype: List[Image] """ - if "brightness" in kwargs: - assert len(kwargs["brightness"]) == len(detectors) - if "contrast" in kwargs: - assert len(kwargs["contrast"]) == len(detectors) + if "brightness" in kwargs and len(kwargs["brightness"]) != len(detectors): + raise ValueError("Number of detectors does not match brightness list") + if "contrast" in kwargs and len(kwargs["contrast"]) != len(detectors): + raise ValueError("Number of detectors does not match contrast list") _ = self.__find_camera(detectors, self.stem_detectors, binning) @@ -674,7 +677,7 @@ def acquire_stem_images(self, self.__check_prerequisites() body = RequestBody(attr="tem.Acquisition", - validator=Image, + validator=List, obj_cls=AcquisitionObj, obj_method="acquire", cameraName=detectors, @@ -684,7 +687,6 @@ def acquire_stem_images(self, return images - def acquire_film(self, film_text: str, exp_time: float) -> None: