Source Code for Module EDShare

  1  # -*- coding: utf8 -*- 
  2  # 
  3  #    Project: The EDNA Kernel 
  4  #             http://www.edna-site.org 
  5  # 
  6  #    File: "$Id:$" 
  7  # 
  8  #    Copyright (C) 2008-2009 European Synchrotron Radiation Facility 
  9  #                            Grenoble, France 
 10  # 
 11  #    Principal authors: Jérôme Kieffer (jerome.kieffer@esrf.fr) 
 12  #  
 13  # 
 14  #    This program is free software: you can redistribute it and/or modify 
 15  #    it under the terms of the GNU Lesser General Public License as published 
 16  #    by the Free Software Foundation, either version 3 of the License, or 
 17  #    (at your option) any later version. 
 18  # 
 19  #    This program is distributed in the hope that it will be useful, 
 20  #    but WITHOUT ANY WARRANTY; without even the implied warranty of 
 21  #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 22  #    GNU Lesser General Public License for more details. 
 23  # 
 24  #    You should have received a copy of the GNU General Public License 
 25  #    and the GNU Lesser General Public License  along with this program.   
 26  #    If not, see <http://www.gnu.org/licenses/>. 
 27  # 
 28   
 29  from __future__ import with_statement 
 30  __authors__ = [ "Jérôme Kieffer" ] 
 31  __contact__ = "jerome.kieffer@esrf.fr" 
 32  __license__ = "LGPLv3+" 
 33  __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" 
 34  __date__ = "20120228" 
 35   
 36  import os, tempfile 
 37  from EDLogging              import EDLogging 
 38  from EDFactoryPluginStatic  import EDFactoryPluginStatic 
 39  from EDUtilsPlatform        import EDUtilsPlatform 
 40  from EDSession              import EDSession 
 41  from EDUtilsPath            import EDUtilsPath 
 42   
 43  TEMPDIR = tempfile.gettempdir() 
 44   
 45  if os.name in ["posix", "nt"]: 
 46      h5pyPath = os.path.join(EDUtilsPath.EDNA_HOME, "libraries", "H5Py-1.3.0", EDUtilsPlatform.architecture) 
 47      h5py = EDFactoryPluginStatic.preImport("h5py", _strPath=h5pyPath, _strForceVersion="1.8", _strMethodVersion="version.api_version") 
 48  #    h5py = None 
 49      if h5py is None: 
 50          BACKEND = "dict" 
 51      else: 
 52          BACKEND = "hdf5" 
 53  else: 
 54      BACKEND = "dict" 
 55  if BACKEND == "dict": 
 56      import pickle 
 57   
 58   
 59 -class EDShare(EDLogging, EDSession): 
 60      """ 
 61      This class implements methods to share (large) arrays between plugins. 
 62       
 63      It is a Singleton   
 64      """ 
 65      _instance = None 
 66 -    def __init__(self): 
 67          EDLogging.__init__(self) 
 68          #EDSession is a static class 
 69          self._backend = BACKEND 
 70          self._filename = None 
 71          self._storage = None 
 72          self._listKeys = [] 
 73          self._dictAttrs = {} #attr are metadata associated to an entry. The values are always cached 
 74   
 75   
 76 -    def __call__(self): 
 77          """ 
 78          Makes this class a SINGLETON 
 79          """ 
 80          return self 
 81   
 82 -    def __getitem__(self, key): 
 83          """ 
 84          getter for a key: 
 85          """ 
 86          try: 
 87              unicode(key) 
 88          except UnicodeDecodeError: 
 89              self.ERROR("EDShare.__getitem__: Unicode Error %s" % key) 
 90          if self.isInitialized(): 
 91              with self.locked(): 
 92                  if key in self._listKeys: 
 93                      try: 
 94                          value = self._storage[key] 
 95                      except Exception: 
 96                          value = None 
 97                          self.ERROR("EDShare (exception):  no such key %s" % key) 
 98                  else: 
 99                      value = None 
100                      self.ERROR("EDShare: no such key %s" % key) 
101          else: 
102              self.WARNING("EDShare is uninitialized: initializing") 
103              self.initialize() 
104              value = None 
105          if value is not None: 
106              return value[:] 
107      get = __getitem__ 
108   
109   
110 -    def __setitem__(self, key, value): 
111          """ 
112          setter for a key/value pair 
113          @type key: string 
114          @type value: int, float, or string ; one or many in lists or arrays (without mixing types)   
115          """ 
116          if not self.isInitialized(): 
117              self.WARNING("EDShare is uninitialized: initializing") 
118              self.initialize() 
119          with self.locked(): 
120              if key in self._listKeys: 
121                  self.ERROR("EDShare: Redefinition of elements is forbidden key=%s, mean value=%s " % (key, value.mean())) 
122              else: 
123                  self._listKeys.append(key) 
124                  self._storage[key] = value 
125                  self._dictAttrs[key] = {} 
126      set = __setitem__ 
127   
128   
129 -    def __contains__(self, key): 
130          return (key in self._listKeys) 
131      has_key = __contains__ 
132   
133   
134 -    def set_metadata(self, key, attr_key, attr_value): 
135          """ 
136          Sets a metadata to an element 
137          """ 
138          if key in self._listKeys: 
139              with self.locked(): 
140                  self._dictAttrs[key][attr_key] = attr_value 
141                  if self._backend == "hdf5": 
142                      self._storage[key].attrs[attr_key] = attr_value 
143          else: 
144              self.ERROR("EDShare: No such element") 
145   
146 -    def get_metadata(self, key, attr_key): 
147          """ 
148          Gets a metadata of an element 
149          """ 
150          if key in self._listKeys: 
151              with self.locked(): 
152                  if attr_key in self._dictAttrs[key]: 
153                      return self._dictAttrs[key][attr_key] 
154                  elif (self._backend == "hdf5") and attr_key in self._storage[key].attrs: 
155                      return self._storage[key].attrs[attr_key] 
156                  else: 
157                      self.ERROR("EDShare: No such Metadata %s in %s" % (key, attr_key)) 
158          else: 
159              self.ERROR("EDShare: No such element: %s" % key) 
160   
161   
162 -    def isInitialized(self): 
163          return not((self._filename is None) and (self._storage is None)) 
164   
165   
166 -    def initialize(self, directory=TEMPDIR, filename=None): 
167          """ 
168          Initialize  EDShare to use this  
169          """ 
170          with self.locked(): 
171              if filename is None: 
172                  if self._backend == "hdf5": 
173                      filename = "EDShare-%s.h5" % self.sessionId 
174                  if self._backend == "dict": 
175                      filename = "EDShare-%s.pickle" % self.sessionId 
176              absFilename = os.path.abspath(os.path.join(directory, filename)) 
177   
178              if not self.isInitialized(): 
179                  self._filename = absFilename 
180                  if not os.path.isdir(directory): 
181                      os.makedirs(directory) 
182                  if (self._backend == "dict"): 
183                      if os.path.isfile(self._filename): 
184                          self._storage = pickle.load(open(self._filename)) 
185                          self._listKeys = self._storage.keys() 
186                          if not isinstance(self._storage, dict): 
187                              self.ERROR("I did not load a dictionary ... resetting") 
188                              self._storage = {} 
189                              self._listKeys = [] 
190                      else: 
191                          self._storage = {} 
192                          self._listKeys = [] 
193                  elif (self._backend == "hdf5"): 
194                      self._storage = h5py.File(self._filename) 
195                      self._storage.visititems(self._analyseHDF5) 
196                  else: 
197                      self.ERROR("unrecognized backend !!!") 
198              else: 
199                  if  (absFilename != self._filename): 
200                      self.ERROR("EDShare was already initialized with backend %s on %s" % (self._backend, self._storage)) 
201   
202   
203 -    def flush(self): 
204          """ 
205          Write the content of the cache on the disk 
206          """ 
207          with self.locked(): 
208              if self.isInitialized(): 
209                  if self._backend == "hdf5": 
210                      if h5py.version.api_version_tuple < (1, 10): 
211                          # This is a work arround because "flush" does not lean a file readable from outside.  
212                          self._storage.close() 
213                          self._storage = h5py.File(self._filename) 
214                      else: 
215                          self._storage.flush() 
216                  elif (self._backend == "dict"): 
217                      fileOut = open(self._filename, "w") 
218                      self._storage = pickle.dump(self._storage, fileOut) 
219                      fileOut.close() 
220                  else: 
221                      self.ERROR("EDShare: unrecognized backend !!!") 
222              else: 
223                      self.ERROR("EDShare: Uninitialized !!!") 
224   
225   
226 -    def close(self, remove=False): 
227          """ 
228          Method that closes the file and resets the cache. 
229          if remove: delete the input file as well.  
230           
231          Useful for testing mainly 
232          """ 
233          with self.locked(): 
234              if self.isInitialized(): 
235                  if self._backend == "hdf5": 
236                      self._storage.close() 
237                  else: 
238                      self._storage.flush() 
239                  self._listKeys = [] 
240                  self._storage = None 
241                  if remove: 
242                      os.unlink(self._filename) 
243                  self._filename = None 
244              else: 
245                  self.ERROR("Closing a file that is uninitialized !!!") 
246   
247   
248 -    def _analyseHDF5(self, name, obj): 
249          """ 
250          Part of the recursive analysis of an HDF5 tree to retrieve all datasets.  
251          Populates self._listKeys 
252           
253          @param name: name of the object itself (string) 
254          @param obj: the h5py object itself 
255          @return None 
256          """ 
257          if isinstance(obj, h5py.Dataset): 
258              self._listKeys.append(name) 
259   
260   
261 -    def items(self): 
262          """ 
263          Returns the list of  
264          """ 
265          return self._listKeys 
266   
267   
268 -    def get_backend(self): 
269          return self._backend 
270      backend = property(get_backend) 
271   
272   
273 -    def get_filename(self): 
274          return self._filename 
275      filename = property(get_filename) 
276   
277   
278  #Make EDShare a singleton 
279  EDShare = EDShare() 
280