1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28 from __future__ import with_statement
29 __authors__ = [ "Jérôme Kieffer", "Olof Svensson" ]
30 __contact__ = "jerome.kieffer@esrf.fr"
31 __license__ = "LGPLv3+"
32 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
33 __date__ = "20120220"
34
35 import base64, hashlib, sys, struct, os
36
37 from EDVerbose import EDVerbose
38 from XSDataCommon import XSDataArray, XSDataString
39 from EDAssert import EDAssert
40 from EDShare import EDShare
41 from EDUtilsPlatform import EDUtilsPlatform
42 from EDFactoryPluginStatic import EDFactoryPluginStatic
43 from EDThreading import Semaphore
44
45 architecture = EDUtilsPlatform.architecture
46 fabioPath = os.path.join(os.environ["EDNA_HOME"], "libraries", "FabIO-0.0.7", architecture)
47 imagingPath = os.path.join(os.environ["EDNA_HOME"], "libraries", "20091115-PIL-1.1.7", architecture)
48 numpyPath = os.path.join(os.environ["EDNA_HOME"], "libraries", "20090405-Numpy-1.3", architecture)
49
50 EDFactoryPluginStatic.preImport("Image", imagingPath)
51 numpy = EDFactoryPluginStatic.preImport("numpy", numpyPath, _strForceVersion="1.2", _strMethodVersion="version.version")
52 fabio = EDFactoryPluginStatic.preImport("fabio", fabioPath, _strForceVersion="0.0.7", _strMethodVersion="version")
53
54 if (numpy is not None) and ("floating" in dir(numpy)):
55 bHaveNumpy = True
56 floatType = (float, numpy.floating)
57 else:
58 try:
59 numpy = __import__("numpy")
60 except ImportError, error:
61 EDVerbose.ERROR("Import numpy failed with error %s" % error)
62 bHaveNumpy = False
63 else:
64 bHaveNumpy = True
65 floatType = (float, numpy.floating)
66
67 if fabio is not None:
68 bHaveFabio = True
69 else:
70 try:
71 fabio = __import__("fabio")
72 except ImportError, error:
73 EDVerbose.ERROR("Import fabio failed with error %s" % error)
74 bHaveFabio = False
75 else:
76 bHaveFabio = True
79 """
80 This is a static utility class for handling numpy like arrays in XML.
81
82 """
83 dTypeSize = {"int64": 8,
84 "uint64": 8,
85 "int32": 4,
86 "uint32": 4,
87 "int16": 2,
88 "uint16": 2,
89 "int8": 1,
90 "uint8": 1,
91 "float32": 4,
92 "float64": 8}
93
94 dTypeFormat = {"int64": "q",
95 "uint64": "Q",
96 "int32": "l",
97 "uint32": "L",
98 "int16": "h",
99 "uint16": "H",
100 "int8": "b",
101 "uint8": "B",
102 "float32": "f",
103 "float64": "d"}
104 semArrayToXSData = Semaphore()
105 semXsDataToArray = Semaphore()
106 semGetArray = Semaphore()
107
108
109 @classmethod
110 - def arrayToXSData(cls, _array, _bIncludeMd5sum=True, _bForceNoNumpy=False, _bUseAsserts=False):
111 """
112 convert a numpy array or a list of list into an XSDataArray object
113 @param _array: numpy array or array-like list
114 @param _bIncludeMd5sum: should the md5sum be added to the XSDataArray instance.
115 It is useful when sending object through the network
116 It is a problem for testing
117 @param _bForceNoNumpy: enables tests without numpy
118 @type includeMd5sum: boolean
119 @return: XSDataArray instance
120 """
121 stringArray = ""
122 shape = None
123 dtype = None
124 sizeDtype = None
125 if bHaveNumpy is True and _bForceNoNumpy is False:
126 EDVerbose.DEBUG("EDUtilsArray.arrayToXSData with numpy")
127
128 if sys.byteorder == "big":
129 _array.byteswap(True)
130 stringArray = _array.tostring()
131 shape = _array.shape
132 dtype = str(_array.dtype)
133 sizeDtype = len(numpy.zeros(1, dtype=_array.dtype).tostring())
134
135 else:
136 EDVerbose.DEBUG("EDUtilsArray.arrayToXSData without numpy")
137 sizeDtype = 8
138 shape = []
139 subarray = _array
140 while True:
141 try:
142 l = len(subarray)
143 except TypeError:
144 break
145 shape.append(l)
146 if l > 0:
147 subarray = subarray[0]
148 else:
149 break
150
151 if len(shape) == 1:
152 if isinstance(_array[0], floatType):
153 dtype = "float64"
154 stringArray = struct.pack("<" + "d" * shape[0], *_array)
155 else:
156 dtype = "int64"
157 stringArray = struct.pack("<" + "l" * shape[0], *_array)
158 elif len(shape) == 2:
159 if isinstance(_array[0][0], floatType):
160 dtype = "float64"
161 lineshape = "<" + "d" * shape[-1]
162 else:
163 dtype = "int64"
164 lineshape = "<" + "q" * shape[-1]
165 for subarray in _array:
166 stringArray += struct.pack(lineshape, *subarray)
167 elif len(shape) == 3:
168 if isinstance(_array[0][0][0], floatType):
169 dtype = "float64"
170 lineshape = "<" + "d" * shape[-1]
171 else:
172 dtype = "int64"
173 lineshape = "<" + "q" * shape[-1]
174 for subarray in _array:
175 for subsubarray in subarray:
176 stringArray += struct.pack(lineshape, *subsubarray)
177 else:
178 EDVerbose.WARNING("EDUtilsArray.arrayToXSDataArray: Array too large %s " % (shape))
179 size = 1
180 for i in shape:
181 size *= i
182
183 xsdArray = XSDataArray(data=base64.b64encode(stringArray),
184 coding=XSDataString("base64"),
185 shape=list(shape),
186 dtype=dtype,
187 size=size)
188 if _bUseAsserts:
189 EDAssert.equal(size * sizeDtype, len(stringArray), "string representing the array has the right size")
190 if _bIncludeMd5sum is True:
191 xsdArray.setMd5sum(XSDataString(hashlib.md5(stringArray).hexdigest()))
192 return xsdArray
193
194
195 @classmethod
196 - def xsDataToArray(cls, _xsdata, _bCheckMd5sum=True, _bForceNoNumpy=False, _bUseAsserts=False):
197 """
198 convert a XSDataArray into either a numpy array or a list of list
199 @param _xsdata: XSDataArray instance
200 @param checkMd5sum: Check if the data are correct based on the checksum
201 It is useful when sending object through the network
202 It is a problem for testing
203 @type _bCheckMd5sum: boolean
204 @param _bForceNoNumpy: enables tests without numpy
205 @type _bForceNoNumpy: boolean
206 @return: numpy array or array-like list depending on what is available on the computer
207 """
208 shape = tuple(_xsdata.getShape())
209 encData = _xsdata.getData()
210 iSize = _xsdata.getSize()
211 dtype = str(_xsdata.getDtype())
212
213 if _xsdata.getCoding() is not None:
214 strCoding = _xsdata.getCoding().getValue()
215 if strCoding == "base64":
216 decData = base64.b64decode(encData)
217 elif strCoding == "base32":
218 decData = base64.b32decode(encData)
219 elif strCoding == "base16":
220 decData = base64.b16decode(encData)
221 else:
222 EDVerbose.WARNING("Unable to recognize the encoding of the data !!! got %s, expected base64, base32 or base16, I assume it is base64 " % strCoding)
223 decData = base64.b64decode(encData)
224 else:
225 EDVerbose.WARNING("No coding provided, I assume it is base64 ")
226 strCoding = "base64"
227 decData = base64.b64decode(encData)
228 EDVerbose.DEBUG("Reading numpy array: len(EncData)= %s, len(decData)=%s, shape=%s, size=%s, dtype=%s, coding=%s" % (len(encData), len(decData), shape, _xsdata.getSize(), _xsdata.getDtype(), strCoding))
229 if _bUseAsserts:
230 EDAssert.equal(len(decData), cls.dTypeSize[dtype] * iSize, "decoded data has the expected size")
231
232 if (_xsdata.getMd5sum() is not None) and (_bCheckMd5sum is True) and _bUseAsserts:
233 EDAssert.equal(_xsdata.getMd5sum().getValue() , hashlib.md5(decData).hexdigest(), "md5sum is correct")
234
235 if bHaveNumpy is True and _bForceNoNumpy is False:
236 EDVerbose.DEBUG("EDUtilsArray.xsDataToArray with numpy")
237 try:
238 matIn = numpy.fromstring(decData, dtype=_xsdata.getDtype())
239 except Exception:
240 matIn = numpy.fromstring(decData, dtype=numpy.dtype(str(_xsdata.getDtype())))
241 arrayOut = matIn.reshape(shape)
242
243 if sys.byteorder == "big":
244 arrayOut.byteswap(True)
245 else:
246 EDVerbose.DEBUG("EDUtilsArray.xsDataToArray without numpy")
247 arrayOut = []
248 lineshape = "<" + cls.dTypeFormat[dtype] * shape[-1]
249 if len(shape) == 1:
250 arrayOut = list(struct.unpack(lineshape, decData))
251 elif len(shape) == 2:
252 linesize = shape[-1] * cls.dTypeSize[dtype]
253 for i in xrange(shape[0]):
254 arrayOut.append(list(struct.unpack(lineshape, decData[i * linesize :(i + 1) * linesize ])))
255 elif len(shape) == 3:
256 linesize = shape[-1] * cls.dTypeSize[dtype]
257 imgsize = shape[-2] * linesize
258 for i in xrange(shape[0]):
259 subarray = []
260 for j in xrange(shape[1]):
261 subarray.append(list(struct.unpack(lineshape, decData[imgsize * i + j * linesize :imgsize * i + (j + 1) * linesize ])))
262 arrayOut.append(subarray)
263 return arrayOut
264
265
266 @classmethod
268 """
269 Tries to retrieve a numpy array from an Image or from a shared array or an XSD object
270
271 As this method returns a numpy array, this method works only on plateform where numpy is available.
272
273 @param _inputObject: XSDataArray or XSDataImageExt or XSDataFile
274 @return: numpy ndarray
275 """
276 npaOutput = None
277 bError = False
278 lstAttrObj = dir(_inputObject)
279 if "array" in lstAttrObj and _inputObject.array is not None:
280 npaOutput = cls.xsDataToArray(_inputObject.array)
281 elif "shared" in lstAttrObj and _inputObject.shared is not None:
282 npaOutput = EDShare[_inputObject.shared.value]
283 elif "path" in lstAttrObj and(_inputObject.path is not None) and os.path.isfile(_inputObject.path.value):
284 if bHaveFabio is True:
285 try:
286 npaOutput = fabio.open(_inputObject.path.value).data
287 except Exception:
288 bError = True
289 else:
290 bError = True
291 elif "XSDataArray" in str(type(_inputObject)):
292 npaOutput = cls.xsDataToArray(_inputObject)
293 if bError is True:
294 EDVerbose.ERROR("EDUtilsArray.getArray works better on platform with numpy & fabio ... No solution found for you, sorry.%s%s " % (os.linesep, _inputObject.marshal()))
295 return npaOutput
296