CYCLUS
hdf5_back.cc
Go to the documentation of this file.
1 #include "hdf5_back.h"
2 
3 #include <cmath>
4 #include <string.h>
5 #include <iostream>
6 
7 #include "blob.h"
8 
9 namespace cyclus {
10 
11 Hdf5Back::Hdf5Back(std::string path) : path_(path) {
12  H5open();
13  hasher_.Clear();
14  if (boost::filesystem::exists(path_))
15  file_ = H5Fopen(path_.c_str(), H5F_ACC_RDWR, H5P_DEFAULT);
16  else
17  file_ = H5Fcreate(path_.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
18  opened_types_.clear();
19  vldatasets_.clear();
20  vldts_.clear();
21  vlkeys_.clear();
22 
23  uuid_type_ = H5Tcopy(H5T_C_S1);
24  H5Tset_size(uuid_type_, CYCLUS_UUID_SIZE);
25  H5Tset_strpad(uuid_type_, H5T_STR_NULLPAD);
26  opened_types_.insert(uuid_type_);
27 
28  hsize_t sha1_len = CYCLUS_SHA1_NINT; // 160 bits == 32 bits / int * 5 ints
29  sha1_type_ = H5Tarray_create2(H5T_NATIVE_UINT, 1, &sha1_len);
30  opened_types_.insert(sha1_type_);
31 
32  vlstr_type_ = H5Tcopy(H5T_C_S1);
33  H5Tset_size(vlstr_type_, H5T_VARIABLE);
34  opened_types_.insert(vlstr_type_);
35  vldts_[VL_STRING] = vlstr_type_;
36 
37  blob_type_ = vlstr_type_;
38  vldts_[BLOB] = blob_type_;
39 }
40 
42  // make sure we are still open
43  if (closed_)
44  return;
45 
46  // cleanup HDF5
47  Flush();
48  H5Fclose(file_);
49  std::set<hid_t>::iterator t;
50  for (t = opened_types_.begin(); t != opened_types_.end(); ++t)
51  H5Tclose(*t);
52  std::map<std::string, hid_t>::iterator vldsit;
53  for (vldsit = vldatasets_.begin(); vldsit != vldatasets_.end(); ++vldsit)
54  H5Dclose(vldsit->second);
55 
56  // cleanup memory
57  std::map<std::string, size_t*>::iterator it;
58  std::map<std::string, DbTypes*>::iterator dbtit;
59  for (it = col_offsets_.begin(); it != col_offsets_.end(); ++it) {
60  delete[](it->second);
61  }
62  for (it = col_sizes_.begin(); it != col_sizes_.end(); ++it) {
63  delete[](it->second);
64  }
65  for (dbtit = schemas_.begin(); dbtit != schemas_.end(); ++dbtit) {
66  delete[](dbtit->second);
67  }
68 
69  closed_ = true;
70 }
71 
73  if (!closed_)
74  Close();
75 }
76 
78  std::map<std::string, DatumList> groups;
79  for (DatumList::iterator it = data.begin(); it != data.end(); ++it) {
80  std::string name = (*it)->title();
81  if (schema_sizes_.count(name) == 0) {
82  if (H5Lexists(file_, name.c_str(), H5P_DEFAULT)) {
83  LoadTableTypes(name, (*it)->vals().size());
84  } else {
85  // Datum* d = *it;
86  // CreateTable(d);
87  CreateTable(*it);
88  }
89  }
90  groups[name].push_back(*it);
91  }
92 
93  std::map<std::string, DatumList>::iterator it;
94  for (it = groups.begin(); it != groups.end(); ++it) {
95  WriteGroup(it->second);
96  }
97 }
98 
99 template <>
100 std::string Hdf5Back::VLRead<std::string, VL_STRING>(const char* rawkey) {
101  using std::string;
102  // key is used as offset
103  Digest key;
104  memcpy(key.val, rawkey, CYCLUS_SHA1_SIZE);
105  const std::vector<hsize_t> idx = key.cast<hsize_t>();
106  hid_t dset = VLDataset(VL_STRING, false);
107  hid_t dspace = H5Dget_space(dset);
108  hid_t mspace = H5Screate_simple(CYCLUS_SHA1_NINT, vlchunk_, NULL);
109  herr_t status = H5Sselect_hyperslab(dspace, H5S_SELECT_SET,
110  (const hsize_t*) &idx[0],
111  NULL, vlchunk_, NULL);
112  if (status < 0)
113  throw IOError("could not select hyperslab of string value array for reading "
114  "in the database '" + path_ + "'.");
115  char** buf = new char*[sizeof(char *)];
116  status = H5Dread(dset, vldts_[VL_STRING], mspace, dspace, H5P_DEFAULT, buf);
117  if (status < 0)
118  throw IOError("failed to read in variable length string data "
119  "in database '" + path_ + "'.");
120  string val;
121  if (buf[0] != NULL)
122  val = string(buf[0]);
123  status = H5Dvlen_reclaim(vldts_[VL_STRING], mspace, H5P_DEFAULT, buf);
124  if (status < 0)
125  throw IOError("failed to reclaim variable length string data space in "
126  "database '" + path_ + "'.");
127  delete[] buf;
128  H5Sclose(mspace);
129  H5Sclose(dspace);
130  return val;
131 }
132 
133 template <>
134 Blob Hdf5Back::VLRead<Blob, BLOB>(const char* rawkey) {
135  // key is used as offset
136  Digest key;
137  memcpy(key.val, rawkey, CYCLUS_SHA1_SIZE);
138  const std::vector<hsize_t> idx = key.cast<hsize_t>();
139  hid_t dset = VLDataset(BLOB, false);
140  hid_t dspace = H5Dget_space(dset);
141  hid_t mspace = H5Screate_simple(CYCLUS_SHA1_NINT, vlchunk_, NULL);
142  herr_t status = H5Sselect_hyperslab(dspace, H5S_SELECT_SET,
143  (const hsize_t*) &idx[0],
144  NULL, vlchunk_, NULL);
145  if (status < 0)
146  throw IOError("could not select hyperslab of Blob value array for reading "
147  "in the database '" + path_ + "'.");
148  char** buf = new char*[sizeof(char *)];
149  status = H5Dread(dset, vldts_[BLOB], mspace, dspace, H5P_DEFAULT, buf);
150  if (status < 0)
151  throw IOError("failed to read in Blob data in database '" + path_ + "'.");
152  Blob val = Blob(buf[0]);
153  status = H5Dvlen_reclaim(vldts_[BLOB], mspace, H5P_DEFAULT, buf);
154  if (status < 0)
155  throw IOError("failed to reclaim Blob data space in database "
156  "'" + path_ + "'.");
157  delete[] buf;
158  H5Sclose(mspace);
159  H5Sclose(dspace);
160  return val;
161 }
162 
163 QueryResult Hdf5Back::Query(std::string table, std::vector<Cond>* conds) {
164  using std::string;
165  using std::vector;
166  using std::set;
167  using std::list;
168  using std::pair;
169  using std::map;
170  if (!H5Lexists(file_, table.c_str(), H5P_DEFAULT))
171  throw IOError("table '" + table + "' does not exist in '" + path_ + "'.");
172  int i;
173  int j;
174  int jlen;
175  herr_t status = 0;
176  hid_t tb_set = H5Dopen2(file_, table.c_str(), H5P_DEFAULT);
177  hid_t tb_space = H5Dget_space(tb_set);
178  hid_t tb_plist = H5Dget_create_plist(tb_set);
179  hid_t tb_type = H5Dget_type(tb_set);
180  size_t tb_typesize = H5Tget_size(tb_type);
181  int tb_length = H5Sget_simple_extent_npoints(tb_space);
182  hsize_t tb_chunksize;
183  H5Pget_chunk(tb_plist, 1, &tb_chunksize);
184  unsigned int nchunks =
185  (tb_length/tb_chunksize) + (tb_length%tb_chunksize == 0?0:1);
186 
187  // set up field-conditions map
188  std::map<std::string, std::vector<Cond*> > field_conds =
189  std::map<std::string, std::vector<Cond*> >();
190  if (conds != NULL) {
191  Cond* cond;
192  for (i = 0; i < conds->size(); ++i) {
193  cond = &((*conds)[i]);
194  if (field_conds.count(cond->field) == 0)
195  field_conds[cond->field] = std::vector<Cond*>();
196  field_conds[cond->field].push_back(cond);
197  }
198  }
199 
200  // read in data
201  QueryResult qr = GetTableInfo(table, tb_set, tb_type);
202  int nfields = qr.fields.size();
203  for (i = 0; i < nfields; ++i) {
204  if (field_conds.count(qr.fields[i]) == 0) {
205  field_conds[qr.fields[i]] = std::vector<Cond*>();
206  }
207  }
208  for (unsigned int n = 0; n < nchunks; ++n) {
209  // This loop is meant to be OpenMP-izable
210  hid_t field_type;
211  hsize_t start = n * tb_chunksize;
212  hsize_t count =
213  (tb_length-start) < tb_chunksize ? tb_length - start : tb_chunksize;
214  char* buf = new char[tb_typesize * count];
215  hid_t memspace = H5Screate_simple(1, &count, NULL);
216  status = H5Sselect_hyperslab(tb_space, H5S_SELECT_SET, &start, NULL,
217  &count, NULL);
218  status = H5Dread(tb_set, tb_type, memspace, tb_space, H5P_DEFAULT, buf);
219  int offset = 0;
220  bool is_row_selected;
221  for (i = 0; i < count; ++i) {
222  offset = i * tb_typesize;
223  is_row_selected = true;
224  QueryRow row = QueryRow(nfields);
225  for (j = 0; j < nfields; ++j) {
226  switch (qr.types[j]) {
227  case VECTOR_INT: {
228  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
229  unsigned int total_size0=H5Tget_size(fieldtype0);
230  hsize_t fieldlen0;
231  H5Tget_array_dims2(fieldtype0,&fieldlen0);
232  hid_t item_type0=H5Tget_super(fieldtype0);
233  unsigned int total_size1elem=H5Tget_size(item_type0);
234  std::vector<int> x0;
235  x0=std::vector<int>(fieldlen0);
236  memcpy(&x0[0],buf+offset,total_size0);
237  is_row_selected=CmpConds<std::vector<int>>(&x0,&(field_conds[qr.fields[j]]));
238  if(is_row_selected){
239  row[j]=x0;
240 
241  }
242  H5Tclose(item_type0);
243  H5Tclose(fieldtype0);
244  break;
245 
246  }
247  case VL_SET_INT: {
248  std::set<int> x0;
249  x0=VLRead<std::set<int>,VL_SET_INT>(buf+offset);
250  is_row_selected=CmpConds<std::set<int>>(&x0,&(field_conds[qr.fields[j]]));
251  if(is_row_selected){
252  row[j]=x0;
253 
254  }
255  break;
256 
257  }
258  case MAP_VL_STRING_STRING: {
259  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
260  unsigned int total_size0=H5Tget_size(fieldtype0);
261  hsize_t fieldlen0;
262  H5Tget_array_dims2(fieldtype0,&fieldlen0);
263  hid_t item_type0=H5Tget_super(fieldtype0);
264  hid_t fieldtype1key=H5Tget_member_type(item_type0,0);
265  unsigned int total_size1key=H5Tget_size(fieldtype1key);
266  size_t nullpos1val;
267  hid_t fieldtype1val=H5Tget_member_type(item_type0,1);
268  unsigned int total_size1val=H5Tget_size(fieldtype1val);
269  std::map<std::string, std::string> x0;
270  for(unsigned int k0=0;k0<fieldlen0;++k0){
271  std::string x1key;
272  x1key=VLRead<std::string,VL_STRING>(buf+offset+(total_size1key+total_size1val)*k0);
273  std::string x1val;
274  x1val=std::string(buf+offset+(total_size1key+total_size1val)*k0+total_size1key,total_size1val);
275  nullpos1val=x1val.find('\0');
276  if(nullpos1val!=std::string::npos){
277  x1val.resize(nullpos1val);
278 
279  }
280  x0[x1key]=x1val;
281 
282  }
283  is_row_selected=CmpConds<std::map<std::string, std::string>>(&x0,&(field_conds[qr.fields[j]]));
284  if(is_row_selected){
285  row[j]=x0;
286 
287  }
288  H5Tclose(fieldtype1val);
289  H5Tclose(fieldtype1key);
290  H5Tclose(item_type0);
291  H5Tclose(fieldtype0);
292  break;
293 
294  }
295  case MAP_INT_VL_STRING: {
296  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
297  unsigned int total_size0=H5Tget_size(fieldtype0);
298  hsize_t fieldlen0;
299  H5Tget_array_dims2(fieldtype0,&fieldlen0);
300  hid_t item_type0=H5Tget_super(fieldtype0);
301  hid_t fieldtype1key=H5Tget_member_type(item_type0,0);
302  unsigned int total_size1key=H5Tget_size(fieldtype1key);
303  hid_t fieldtype1val=H5Tget_member_type(item_type0,1);
304  unsigned int total_size1val=H5Tget_size(fieldtype1val);
305  std::map<int, std::string> x0;
306  for(unsigned int k0=0;k0<fieldlen0;++k0){
307  int x1key;
308  x1key=*reinterpret_cast<int*>(buf+offset+(total_size1key+total_size1val)*k0);
309  std::string x1val;
310  x1val=VLRead<std::string,VL_STRING>(buf+offset+(total_size1key+total_size1val)*k0+total_size1key);
311  x0[x1key]=x1val;
312 
313  }
314  is_row_selected=CmpConds<std::map<int, std::string>>(&x0,&(field_conds[qr.fields[j]]));
315  if(is_row_selected){
316  row[j]=x0;
317 
318  }
319  H5Tclose(fieldtype1val);
320  H5Tclose(fieldtype1key);
321  H5Tclose(item_type0);
322  H5Tclose(fieldtype0);
323  break;
324 
325  }
326  case VL_MAP_STRING_DOUBLE: {
327  std::map<std::string, double> x0;
328  x0=VLRead<std::map<std::string, double>,VL_MAP_STRING_DOUBLE>(buf+offset);
329  is_row_selected=CmpConds<std::map<std::string, double>>(&x0,&(field_conds[qr.fields[j]]));
330  if(is_row_selected){
331  row[j]=x0;
332 
333  }
334  break;
335 
336  }
337  case VL_MAP_INT_DOUBLE: {
338  std::map<int, double> x0;
339  x0=VLRead<std::map<int, double>,VL_MAP_INT_DOUBLE>(buf+offset);
340  is_row_selected=CmpConds<std::map<int, double>>(&x0,&(field_conds[qr.fields[j]]));
341  if(is_row_selected){
342  row[j]=x0;
343 
344  }
345  break;
346 
347  }
348  case LIST_VL_STRING: {
349  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
350  unsigned int total_size0=H5Tget_size(fieldtype0);
351  hsize_t fieldlen0;
352  H5Tget_array_dims2(fieldtype0,&fieldlen0);
353  hid_t item_type0=H5Tget_super(fieldtype0);
354  unsigned int total_size1elem=H5Tget_size(item_type0);
355  std::list<std::string> x0;
356  for(unsigned int k0=0;k0<fieldlen0;++k0){
357  std::string x1elem;
358  x1elem=VLRead<std::string,VL_STRING>(buf+offset+total_size1elem*k0);
359  x0.push_back(x1elem);
360 
361  }
362  is_row_selected=CmpConds<std::list<std::string>>(&x0,&(field_conds[qr.fields[j]]));
363  if(is_row_selected){
364  row[j]=x0;
365 
366  }
367  H5Tclose(item_type0);
368  H5Tclose(fieldtype0);
369  break;
370 
371  }
372  case VL_VECTOR_STRING: {
373  std::vector<std::string> x0;
374  x0=VLRead<std::vector<std::string>,VL_VECTOR_STRING>(buf+offset);
375  is_row_selected=CmpConds<std::vector<std::string>>(&x0,&(field_conds[qr.fields[j]]));
376  if(is_row_selected){
377  row[j]=x0;
378 
379  }
380  break;
381 
382  }
383  case VECTOR_STRING: {
384  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
385  unsigned int total_size0=H5Tget_size(fieldtype0);
386  hsize_t fieldlen0;
387  H5Tget_array_dims2(fieldtype0,&fieldlen0);
388  hid_t item_type0=H5Tget_super(fieldtype0);
389  size_t nullpos1elem;
390  unsigned int total_size1elem=H5Tget_size(item_type0);
391  std::vector<std::string> x0;
392  x0=std::vector<std::string>(fieldlen0);
393  for(unsigned int k0=0;k0<fieldlen0;++k0){
394  x0[k0]=std::string(buf+offset+total_size1elem*k0,total_size1elem);
395  nullpos1elem=x0[k0].find('\0');
396  if(nullpos1elem!=std::string::npos){
397  x0[k0].resize(nullpos1elem);
398 
399  }
400 
401  }
402  is_row_selected=CmpConds<std::vector<std::string>>(&x0,&(field_conds[qr.fields[j]]));
403  if(is_row_selected){
404  row[j]=x0;
405 
406  }
407  H5Tclose(item_type0);
408  H5Tclose(fieldtype0);
409  break;
410 
411  }
412  case PAIR_INT_STRING: {
413  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
414  unsigned int total_size0=H5Tget_size(fieldtype0);
415  hid_t fieldtype1first=H5Tget_member_type(fieldtype0,0);
416  unsigned int total_size1first=H5Tget_size(fieldtype1first);
417  size_t nullpos1second;
418  hid_t fieldtype1second=H5Tget_member_type(fieldtype0,1);
419  unsigned int total_size1second=H5Tget_size(fieldtype1second);
420  std::pair<int, std::string> x0;
421  int x1first;
422  x1first=*reinterpret_cast<int*>(buf+offset);
423  std::string x1second;
424  x1second=std::string(buf+offset+total_size1first,total_size1second);
425  nullpos1second=x1second.find('\0');
426  if(nullpos1second!=std::string::npos){
427  x1second.resize(nullpos1second);
428 
429  }
430  x0=std::make_pair(x1first,x1second);
431  is_row_selected=CmpConds<std::pair<int, std::string>>(&x0,&(field_conds[qr.fields[j]]));
432  if(is_row_selected){
433  row[j]=x0;
434 
435  }
436  H5Tclose(fieldtype1second);
437  H5Tclose(fieldtype1first);
438  H5Tclose(fieldtype0);
439  break;
440 
441  }
442  case VL_LIST_INT: {
443  std::list<int> x0;
444  x0=VLRead<std::list<int>,VL_LIST_INT>(buf+offset);
445  is_row_selected=CmpConds<std::list<int>>(&x0,&(field_conds[qr.fields[j]]));
446  if(is_row_selected){
447  row[j]=x0;
448 
449  }
450  break;
451 
452  }
454  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
455  unsigned int total_size0=H5Tget_size(fieldtype0);
456  hsize_t fieldlen0;
457  H5Tget_array_dims2(fieldtype0,&fieldlen0);
458  hid_t item_type0=H5Tget_super(fieldtype0);
459  hid_t fieldtype1key=H5Tget_member_type(item_type0,0);
460  unsigned int total_size1key=H5Tget_size(fieldtype1key);
461  hid_t fieldtype2keyfirst=H5Tget_member_type(fieldtype1key,0);
462  unsigned int total_size2keyfirst=H5Tget_size(fieldtype2keyfirst);
463  size_t nullpos2keysecond;
464  hid_t fieldtype2keysecond=H5Tget_member_type(fieldtype1key,1);
465  unsigned int total_size2keysecond=H5Tget_size(fieldtype2keysecond);
466  hid_t fieldtype1val=H5Tget_member_type(item_type0,1);
467  unsigned int total_size1val=H5Tget_size(fieldtype1val);
468  std::map<std::pair<int, std::string>, double> x0;
469  for(unsigned int k0=0;k0<fieldlen0;++k0){
470  std::pair<int, std::string> x1key;
471  int x2keyfirst;
472  x2keyfirst=*reinterpret_cast<int*>(buf+offset+(total_size1key+total_size1val)*k0);
473  std::string x2keysecond;
474  x2keysecond=std::string(buf+offset+(total_size1key+total_size1val)*k0+total_size2keyfirst,total_size2keysecond);
475  nullpos2keysecond=x2keysecond.find('\0');
476  if(nullpos2keysecond!=std::string::npos){
477  x2keysecond.resize(nullpos2keysecond);
478 
479  }
480  x1key=std::make_pair(x2keyfirst,x2keysecond);
481  double x1val;
482  x1val=*reinterpret_cast<double*>(buf+offset+(total_size1key+total_size1val)*k0+total_size1key);
483  x0[x1key]=x1val;
484 
485  }
486  is_row_selected=CmpConds<std::map<std::pair<int, std::string>, double>>(&x0,&(field_conds[qr.fields[j]]));
487  if(is_row_selected){
488  row[j]=x0;
489 
490  }
491  H5Tclose(fieldtype1val);
492  H5Tclose(fieldtype2keysecond);
493  H5Tclose(fieldtype2keyfirst);
494  H5Tclose(fieldtype1key);
495  H5Tclose(item_type0);
496  H5Tclose(fieldtype0);
497  break;
498 
499  }
500  case VL_VECTOR_INT: {
501  std::vector<int> x0;
502  x0=VLRead<std::vector<int>,VL_VECTOR_INT>(buf+offset);
503  is_row_selected=CmpConds<std::vector<int>>(&x0,&(field_conds[qr.fields[j]]));
504  if(is_row_selected){
505  row[j]=x0;
506 
507  }
508  break;
509 
510  }
511  case VL_MAP_INT_INT: {
512  std::map<int, int> x0;
513  x0=VLRead<std::map<int, int>,VL_MAP_INT_INT>(buf+offset);
514  is_row_selected=CmpConds<std::map<int, int>>(&x0,&(field_conds[qr.fields[j]]));
515  if(is_row_selected){
516  row[j]=x0;
517 
518  }
519  break;
520 
521  }
522  case VECTOR_FLOAT: {
523  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
524  unsigned int total_size0=H5Tget_size(fieldtype0);
525  hsize_t fieldlen0;
526  H5Tget_array_dims2(fieldtype0,&fieldlen0);
527  hid_t item_type0=H5Tget_super(fieldtype0);
528  unsigned int total_size1elem=H5Tget_size(item_type0);
529  std::vector<float> x0;
530  x0=std::vector<float>(fieldlen0);
531  memcpy(&x0[0],buf+offset,total_size0);
532  is_row_selected=CmpConds<std::vector<float>>(&x0,&(field_conds[qr.fields[j]]));
533  if(is_row_selected){
534  row[j]=x0;
535 
536  }
537  H5Tclose(item_type0);
538  H5Tclose(fieldtype0);
539  break;
540 
541  }
543  std::map<std::string, std::string> x0;
544  x0=VLRead<std::map<std::string, std::string>,VL_MAP_STRING_VL_STRING>(buf+offset);
545  is_row_selected=CmpConds<std::map<std::string, std::string>>(&x0,&(field_conds[qr.fields[j]]));
546  if(is_row_selected){
547  row[j]=x0;
548 
549  }
550  break;
551 
552  }
553  case MAP_STRING_INT: {
554  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
555  unsigned int total_size0=H5Tget_size(fieldtype0);
556  hsize_t fieldlen0;
557  H5Tget_array_dims2(fieldtype0,&fieldlen0);
558  hid_t item_type0=H5Tget_super(fieldtype0);
559  size_t nullpos1key;
560  hid_t fieldtype1key=H5Tget_member_type(item_type0,0);
561  unsigned int total_size1key=H5Tget_size(fieldtype1key);
562  hid_t fieldtype1val=H5Tget_member_type(item_type0,1);
563  unsigned int total_size1val=H5Tget_size(fieldtype1val);
564  std::map<std::string, int> x0;
565  for(unsigned int k0=0;k0<fieldlen0;++k0){
566  std::string x1key;
567  x1key=std::string(buf+offset+(total_size1key+total_size1val)*k0,total_size1key);
568  nullpos1key=x1key.find('\0');
569  if(nullpos1key!=std::string::npos){
570  x1key.resize(nullpos1key);
571 
572  }
573  int x1val;
574  x1val=*reinterpret_cast<int*>(buf+offset+(total_size1key+total_size1val)*k0+total_size1key);
575  x0[x1key]=x1val;
576 
577  }
578  is_row_selected=CmpConds<std::map<std::string, int>>(&x0,&(field_conds[qr.fields[j]]));
579  if(is_row_selected){
580  row[j]=x0;
581 
582  }
583  H5Tclose(fieldtype1val);
584  H5Tclose(fieldtype1key);
585  H5Tclose(item_type0);
586  H5Tclose(fieldtype0);
587  break;
588 
589  }
590  case VL_SET_STRING: {
591  std::set<std::string> x0;
592  x0=VLRead<std::set<std::string>,VL_SET_STRING>(buf+offset);
593  is_row_selected=CmpConds<std::set<std::string>>(&x0,&(field_conds[qr.fields[j]]));
594  if(is_row_selected){
595  row[j]=x0;
596 
597  }
598  break;
599 
600  }
601  case PAIR_INT_VL_STRING: {
602  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
603  unsigned int total_size0=H5Tget_size(fieldtype0);
604  hid_t fieldtype1first=H5Tget_member_type(fieldtype0,0);
605  unsigned int total_size1first=H5Tget_size(fieldtype1first);
606  hid_t fieldtype1second=H5Tget_member_type(fieldtype0,1);
607  unsigned int total_size1second=H5Tget_size(fieldtype1second);
608  std::pair<int, std::string> x0;
609  int x1first;
610  x1first=*reinterpret_cast<int*>(buf+offset);
611  std::string x1second;
612  x1second=VLRead<std::string,VL_STRING>(buf+offset+total_size1first);
613  x0=std::make_pair(x1first,x1second);
614  is_row_selected=CmpConds<std::pair<int, std::string>>(&x0,&(field_conds[qr.fields[j]]));
615  if(is_row_selected){
616  row[j]=x0;
617 
618  }
619  H5Tclose(fieldtype1second);
620  H5Tclose(fieldtype1first);
621  H5Tclose(fieldtype0);
622  break;
623 
624  }
625  case VL_STRING: {
626  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
627  unsigned int total_size0=H5Tget_size(fieldtype0);
628  std::string x0;
629  x0=VLRead<std::string,VL_STRING>(buf+offset);
630  is_row_selected=CmpConds<std::string>(&x0,&(field_conds[qr.fields[j]]));
631  if(is_row_selected){
632  row[j]=x0;
633 
634  }
635  H5Tclose(fieldtype0);
636  break;
637 
638  }
639  case MAP_STRING_VL_STRING: {
640  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
641  unsigned int total_size0=H5Tget_size(fieldtype0);
642  hsize_t fieldlen0;
643  H5Tget_array_dims2(fieldtype0,&fieldlen0);
644  hid_t item_type0=H5Tget_super(fieldtype0);
645  size_t nullpos1key;
646  hid_t fieldtype1key=H5Tget_member_type(item_type0,0);
647  unsigned int total_size1key=H5Tget_size(fieldtype1key);
648  hid_t fieldtype1val=H5Tget_member_type(item_type0,1);
649  unsigned int total_size1val=H5Tget_size(fieldtype1val);
650  std::map<std::string, std::string> x0;
651  for(unsigned int k0=0;k0<fieldlen0;++k0){
652  std::string x1key;
653  x1key=std::string(buf+offset+(total_size1key+total_size1val)*k0,total_size1key);
654  nullpos1key=x1key.find('\0');
655  if(nullpos1key!=std::string::npos){
656  x1key.resize(nullpos1key);
657 
658  }
659  std::string x1val;
660  x1val=VLRead<std::string,VL_STRING>(buf+offset+(total_size1key+total_size1val)*k0+total_size1key);
661  x0[x1key]=x1val;
662 
663  }
664  is_row_selected=CmpConds<std::map<std::string, std::string>>(&x0,&(field_conds[qr.fields[j]]));
665  if(is_row_selected){
666  row[j]=x0;
667 
668  }
669  H5Tclose(fieldtype1val);
670  H5Tclose(fieldtype1key);
671  H5Tclose(item_type0);
672  H5Tclose(fieldtype0);
673  break;
674 
675  }
677  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
678  unsigned int total_size0=H5Tget_size(fieldtype0);
679  hsize_t fieldlen0;
680  H5Tget_array_dims2(fieldtype0,&fieldlen0);
681  hid_t item_type0=H5Tget_super(fieldtype0);
682  hid_t fieldtype1key=H5Tget_member_type(item_type0,0);
683  unsigned int total_size1key=H5Tget_size(fieldtype1key);
684  hid_t fieldtype2keyfirst=H5Tget_member_type(fieldtype1key,0);
685  unsigned int total_size2keyfirst=H5Tget_size(fieldtype2keyfirst);
686  hid_t fieldtype2keysecond=H5Tget_member_type(fieldtype1key,1);
687  unsigned int total_size2keysecond=H5Tget_size(fieldtype2keysecond);
688  hid_t fieldtype1val=H5Tget_member_type(item_type0,1);
689  unsigned int total_size1val=H5Tget_size(fieldtype1val);
690  std::map<std::pair<int, std::string>, double> x0;
691  for(unsigned int k0=0;k0<fieldlen0;++k0){
692  std::pair<int, std::string> x1key;
693  int x2keyfirst;
694  x2keyfirst=*reinterpret_cast<int*>(buf+offset+(total_size1key+total_size1val)*k0);
695  std::string x2keysecond;
696  x2keysecond=VLRead<std::string,VL_STRING>(buf+offset+(total_size1key+total_size1val)*k0+total_size2keyfirst);
697  x1key=std::make_pair(x2keyfirst,x2keysecond);
698  double x1val;
699  x1val=*reinterpret_cast<double*>(buf+offset+(total_size1key+total_size1val)*k0+total_size1key);
700  x0[x1key]=x1val;
701 
702  }
703  is_row_selected=CmpConds<std::map<std::pair<int, std::string>, double>>(&x0,&(field_conds[qr.fields[j]]));
704  if(is_row_selected){
705  row[j]=x0;
706 
707  }
708  H5Tclose(fieldtype1val);
709  H5Tclose(fieldtype2keysecond);
710  H5Tclose(fieldtype2keyfirst);
711  H5Tclose(fieldtype1key);
712  H5Tclose(item_type0);
713  H5Tclose(fieldtype0);
714  break;
715 
716  }
717  case VL_MAP_INT_VL_STRING: {
718  std::map<int, std::string> x0;
719  x0=VLRead<std::map<int, std::string>,VL_MAP_INT_VL_STRING>(buf+offset);
720  is_row_selected=CmpConds<std::map<int, std::string>>(&x0,&(field_conds[qr.fields[j]]));
721  if(is_row_selected){
722  row[j]=x0;
723 
724  }
725  break;
726 
727  }
728  case LIST_STRING: {
729  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
730  unsigned int total_size0=H5Tget_size(fieldtype0);
731  hsize_t fieldlen0;
732  H5Tget_array_dims2(fieldtype0,&fieldlen0);
733  hid_t item_type0=H5Tget_super(fieldtype0);
734  size_t nullpos1elem;
735  unsigned int total_size1elem=H5Tget_size(item_type0);
736  std::list<std::string> x0;
737  for(unsigned int k0=0;k0<fieldlen0;++k0){
738  std::string x1elem;
739  x1elem=std::string(buf+offset+total_size1elem*k0,total_size1elem);
740  nullpos1elem=x1elem.find('\0');
741  if(nullpos1elem!=std::string::npos){
742  x1elem.resize(nullpos1elem);
743 
744  }
745  x0.push_back(x1elem);
746 
747  }
748  is_row_selected=CmpConds<std::list<std::string>>(&x0,&(field_conds[qr.fields[j]]));
749  if(is_row_selected){
750  row[j]=x0;
751 
752  }
753  H5Tclose(item_type0);
754  H5Tclose(fieldtype0);
755  break;
756 
757  }
758  case MAP_INT_STRING: {
759  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
760  unsigned int total_size0=H5Tget_size(fieldtype0);
761  hsize_t fieldlen0;
762  H5Tget_array_dims2(fieldtype0,&fieldlen0);
763  hid_t item_type0=H5Tget_super(fieldtype0);
764  hid_t fieldtype1key=H5Tget_member_type(item_type0,0);
765  unsigned int total_size1key=H5Tget_size(fieldtype1key);
766  size_t nullpos1val;
767  hid_t fieldtype1val=H5Tget_member_type(item_type0,1);
768  unsigned int total_size1val=H5Tget_size(fieldtype1val);
769  std::map<int, std::string> x0;
770  for(unsigned int k0=0;k0<fieldlen0;++k0){
771  int x1key;
772  x1key=*reinterpret_cast<int*>(buf+offset+(total_size1key+total_size1val)*k0);
773  std::string x1val;
774  x1val=std::string(buf+offset+(total_size1key+total_size1val)*k0+total_size1key,total_size1val);
775  nullpos1val=x1val.find('\0');
776  if(nullpos1val!=std::string::npos){
777  x1val.resize(nullpos1val);
778 
779  }
780  x0[x1key]=x1val;
781 
782  }
783  is_row_selected=CmpConds<std::map<int, std::string>>(&x0,&(field_conds[qr.fields[j]]));
784  if(is_row_selected){
785  row[j]=x0;
786 
787  }
788  H5Tclose(fieldtype1val);
789  H5Tclose(fieldtype1key);
790  H5Tclose(item_type0);
791  H5Tclose(fieldtype0);
792  break;
793 
794  }
795  case VL_MAP_INT_STRING: {
796  std::map<int, std::string> x0;
797  x0=VLRead<std::map<int, std::string>,VL_MAP_INT_STRING>(buf+offset);
798  is_row_selected=CmpConds<std::map<int, std::string>>(&x0,&(field_conds[qr.fields[j]]));
799  if(is_row_selected){
800  row[j]=x0;
801 
802  }
803  break;
804 
805  }
806  case MAP_STRING_DOUBLE: {
807  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
808  unsigned int total_size0=H5Tget_size(fieldtype0);
809  hsize_t fieldlen0;
810  H5Tget_array_dims2(fieldtype0,&fieldlen0);
811  hid_t item_type0=H5Tget_super(fieldtype0);
812  size_t nullpos1key;
813  hid_t fieldtype1key=H5Tget_member_type(item_type0,0);
814  unsigned int total_size1key=H5Tget_size(fieldtype1key);
815  hid_t fieldtype1val=H5Tget_member_type(item_type0,1);
816  unsigned int total_size1val=H5Tget_size(fieldtype1val);
817  std::map<std::string, double> x0;
818  for(unsigned int k0=0;k0<fieldlen0;++k0){
819  std::string x1key;
820  x1key=std::string(buf+offset+(total_size1key+total_size1val)*k0,total_size1key);
821  nullpos1key=x1key.find('\0');
822  if(nullpos1key!=std::string::npos){
823  x1key.resize(nullpos1key);
824 
825  }
826  double x1val;
827  x1val=*reinterpret_cast<double*>(buf+offset+(total_size1key+total_size1val)*k0+total_size1key);
828  x0[x1key]=x1val;
829 
830  }
831  is_row_selected=CmpConds<std::map<std::string, double>>(&x0,&(field_conds[qr.fields[j]]));
832  if(is_row_selected){
833  row[j]=x0;
834 
835  }
836  H5Tclose(fieldtype1val);
837  H5Tclose(fieldtype1key);
838  H5Tclose(item_type0);
839  H5Tclose(fieldtype0);
840  break;
841 
842  }
843  case SET_STRING: {
844  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
845  unsigned int total_size0=H5Tget_size(fieldtype0);
846  hsize_t fieldlen0;
847  H5Tget_array_dims2(fieldtype0,&fieldlen0);
848  hid_t item_type0=H5Tget_super(fieldtype0);
849  size_t nullpos1elem;
850  unsigned int total_size1elem=H5Tget_size(item_type0);
851  std::set<std::string> x0;
852  for(unsigned int k0=0;k0<fieldlen0;++k0){
853  std::string x1elem;
854  x1elem=std::string(buf+offset+total_size1elem*k0,total_size1elem);
855  nullpos1elem=x1elem.find('\0');
856  if(nullpos1elem!=std::string::npos){
857  x1elem.resize(nullpos1elem);
858 
859  }
860  x0.insert(x1elem);
861 
862  }
863  is_row_selected=CmpConds<std::set<std::string>>(&x0,&(field_conds[qr.fields[j]]));
864  if(is_row_selected){
865  row[j]=x0;
866 
867  }
868  H5Tclose(item_type0);
869  H5Tclose(fieldtype0);
870  break;
871 
872  }
874  std::map<std::string, std::string> x0;
875  x0=VLRead<std::map<std::string, std::string>,VL_MAP_VL_STRING_STRING>(buf+offset);
876  is_row_selected=CmpConds<std::map<std::string, std::string>>(&x0,&(field_conds[qr.fields[j]]));
877  if(is_row_selected){
878  row[j]=x0;
879 
880  }
881  break;
882 
883  }
884  case STRING: {
885  size_t nullpos0;
886  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
887  unsigned int total_size0=H5Tget_size(fieldtype0);
888  std::string x0;
889  x0=std::string(buf+offset,total_size0);
890  nullpos0=x0.find('\0');
891  if(nullpos0!=std::string::npos){
892  x0.resize(nullpos0);
893 
894  }
895  is_row_selected=CmpConds<std::string>(&x0,&(field_conds[qr.fields[j]]));
896  if(is_row_selected){
897  row[j]=x0;
898 
899  }
900  H5Tclose(fieldtype0);
901  break;
902 
903  }
904  case VL_SET_VL_STRING: {
905  std::set<std::string> x0;
906  x0=VLRead<std::set<std::string>,VL_SET_VL_STRING>(buf+offset);
907  is_row_selected=CmpConds<std::set<std::string>>(&x0,&(field_conds[qr.fields[j]]));
908  if(is_row_selected){
909  row[j]=x0;
910 
911  }
912  break;
913 
914  }
915  case PAIR_INT_INT: {
916  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
917  unsigned int total_size0=H5Tget_size(fieldtype0);
918  hid_t fieldtype1first=H5Tget_member_type(fieldtype0,0);
919  unsigned int total_size1first=H5Tget_size(fieldtype1first);
920  hid_t fieldtype1second=H5Tget_member_type(fieldtype0,1);
921  unsigned int total_size1second=H5Tget_size(fieldtype1second);
922  std::pair<int, int> x0;
923  int x1first;
924  x1first=*reinterpret_cast<int*>(buf+offset);
925  int x1second;
926  x1second=*reinterpret_cast<int*>(buf+offset+total_size1first);
927  x0=std::make_pair(x1first,x1second);
928  is_row_selected=CmpConds<std::pair<int, int>>(&x0,&(field_conds[qr.fields[j]]));
929  if(is_row_selected){
930  row[j]=x0;
931 
932  }
933  H5Tclose(fieldtype1second);
934  H5Tclose(fieldtype1first);
935  H5Tclose(fieldtype0);
936  break;
937 
938  }
939  case VL_MAP_STRING_STRING: {
940  std::map<std::string, std::string> x0;
941  x0=VLRead<std::map<std::string, std::string>,VL_MAP_STRING_STRING>(buf+offset);
942  is_row_selected=CmpConds<std::map<std::string, std::string>>(&x0,&(field_conds[qr.fields[j]]));
943  if(is_row_selected){
944  row[j]=x0;
945 
946  }
947  break;
948 
949  }
951  std::map<std::pair<int, std::string>, double> x0;
952  x0=VLRead<std::map<std::pair<int, std::string>, double>,VL_MAP_PAIR_INT_VL_STRING_DOUBLE>(buf+offset);
953  is_row_selected=CmpConds<std::map<std::pair<int, std::string>, double>>(&x0,&(field_conds[qr.fields[j]]));
954  if(is_row_selected){
955  row[j]=x0;
956 
957  }
958  break;
959 
960  }
961  case MAP_VL_STRING_DOUBLE: {
962  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
963  unsigned int total_size0=H5Tget_size(fieldtype0);
964  hsize_t fieldlen0;
965  H5Tget_array_dims2(fieldtype0,&fieldlen0);
966  hid_t item_type0=H5Tget_super(fieldtype0);
967  hid_t fieldtype1key=H5Tget_member_type(item_type0,0);
968  unsigned int total_size1key=H5Tget_size(fieldtype1key);
969  hid_t fieldtype1val=H5Tget_member_type(item_type0,1);
970  unsigned int total_size1val=H5Tget_size(fieldtype1val);
971  std::map<std::string, double> x0;
972  for(unsigned int k0=0;k0<fieldlen0;++k0){
973  std::string x1key;
974  x1key=VLRead<std::string,VL_STRING>(buf+offset+(total_size1key+total_size1val)*k0);
975  double x1val;
976  x1val=*reinterpret_cast<double*>(buf+offset+(total_size1key+total_size1val)*k0+total_size1key);
977  x0[x1key]=x1val;
978 
979  }
980  is_row_selected=CmpConds<std::map<std::string, double>>(&x0,&(field_conds[qr.fields[j]]));
981  if(is_row_selected){
982  row[j]=x0;
983 
984  }
985  H5Tclose(fieldtype1val);
986  H5Tclose(fieldtype1key);
987  H5Tclose(item_type0);
988  H5Tclose(fieldtype0);
989  break;
990 
991  }
993  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
994  unsigned int total_size0=H5Tget_size(fieldtype0);
995  hsize_t fieldlen0;
996  H5Tget_array_dims2(fieldtype0,&fieldlen0);
997  hid_t item_type0=H5Tget_super(fieldtype0);
998  hid_t fieldtype1key=H5Tget_member_type(item_type0,0);
999  unsigned int total_size1key=H5Tget_size(fieldtype1key);
1000  hid_t fieldtype1val=H5Tget_member_type(item_type0,1);
1001  unsigned int total_size1val=H5Tget_size(fieldtype1val);
1002  std::map<std::string, std::string> x0;
1003  for(unsigned int k0=0;k0<fieldlen0;++k0){
1004  std::string x1key;
1005  x1key=VLRead<std::string,VL_STRING>(buf+offset+(total_size1key+total_size1val)*k0);
1006  std::string x1val;
1007  x1val=VLRead<std::string,VL_STRING>(buf+offset+(total_size1key+total_size1val)*k0+total_size1key);
1008  x0[x1key]=x1val;
1009 
1010  }
1011  is_row_selected=CmpConds<std::map<std::string, std::string>>(&x0,&(field_conds[qr.fields[j]]));
1012  if(is_row_selected){
1013  row[j]=x0;
1014 
1015  }
1016  H5Tclose(fieldtype1val);
1017  H5Tclose(fieldtype1key);
1018  H5Tclose(item_type0);
1019  H5Tclose(fieldtype0);
1020  break;
1021 
1022  }
1023  case MAP_VL_STRING_INT: {
1024  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
1025  unsigned int total_size0=H5Tget_size(fieldtype0);
1026  hsize_t fieldlen0;
1027  H5Tget_array_dims2(fieldtype0,&fieldlen0);
1028  hid_t item_type0=H5Tget_super(fieldtype0);
1029  hid_t fieldtype1key=H5Tget_member_type(item_type0,0);
1030  unsigned int total_size1key=H5Tget_size(fieldtype1key);
1031  hid_t fieldtype1val=H5Tget_member_type(item_type0,1);
1032  unsigned int total_size1val=H5Tget_size(fieldtype1val);
1033  std::map<std::string, int> x0;
1034  for(unsigned int k0=0;k0<fieldlen0;++k0){
1035  std::string x1key;
1036  x1key=VLRead<std::string,VL_STRING>(buf+offset+(total_size1key+total_size1val)*k0);
1037  int x1val;
1038  x1val=*reinterpret_cast<int*>(buf+offset+(total_size1key+total_size1val)*k0+total_size1key);
1039  x0[x1key]=x1val;
1040 
1041  }
1042  is_row_selected=CmpConds<std::map<std::string, int>>(&x0,&(field_conds[qr.fields[j]]));
1043  if(is_row_selected){
1044  row[j]=x0;
1045 
1046  }
1047  H5Tclose(fieldtype1val);
1048  H5Tclose(fieldtype1key);
1049  H5Tclose(item_type0);
1050  H5Tclose(fieldtype0);
1051  break;
1052 
1053  }
1054  case VL_LIST_VL_STRING: {
1055  std::list<std::string> x0;
1056  x0=VLRead<std::list<std::string>,VL_LIST_VL_STRING>(buf+offset);
1057  is_row_selected=CmpConds<std::list<std::string>>(&x0,&(field_conds[qr.fields[j]]));
1058  if(is_row_selected){
1059  row[j]=x0;
1060 
1061  }
1062  break;
1063 
1064  }
1065  case SET_VL_STRING: {
1066  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
1067  unsigned int total_size0=H5Tget_size(fieldtype0);
1068  hsize_t fieldlen0;
1069  H5Tget_array_dims2(fieldtype0,&fieldlen0);
1070  hid_t item_type0=H5Tget_super(fieldtype0);
1071  unsigned int total_size1elem=H5Tget_size(item_type0);
1072  std::set<std::string> x0;
1073  for(unsigned int k0=0;k0<fieldlen0;++k0){
1074  std::string x1elem;
1075  x1elem=VLRead<std::string,VL_STRING>(buf+offset+total_size1elem*k0);
1076  x0.insert(x1elem);
1077 
1078  }
1079  is_row_selected=CmpConds<std::set<std::string>>(&x0,&(field_conds[qr.fields[j]]));
1080  if(is_row_selected){
1081  row[j]=x0;
1082 
1083  }
1084  H5Tclose(item_type0);
1085  H5Tclose(fieldtype0);
1086  break;
1087 
1088  }
1089  case VL_MAP_STRING_INT: {
1090  std::map<std::string, int> x0;
1091  x0=VLRead<std::map<std::string, int>,VL_MAP_STRING_INT>(buf+offset);
1092  is_row_selected=CmpConds<std::map<std::string, int>>(&x0,&(field_conds[qr.fields[j]]));
1093  if(is_row_selected){
1094  row[j]=x0;
1095 
1096  }
1097  break;
1098 
1099  }
1100  case VECTOR_DOUBLE: {
1101  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
1102  unsigned int total_size0=H5Tget_size(fieldtype0);
1103  hsize_t fieldlen0;
1104  H5Tget_array_dims2(fieldtype0,&fieldlen0);
1105  hid_t item_type0=H5Tget_super(fieldtype0);
1106  unsigned int total_size1elem=H5Tget_size(item_type0);
1107  std::vector<double> x0;
1108  x0=std::vector<double>(fieldlen0);
1109  memcpy(&x0[0],buf+offset,total_size0);
1110  is_row_selected=CmpConds<std::vector<double>>(&x0,&(field_conds[qr.fields[j]]));
1111  if(is_row_selected){
1112  row[j]=x0;
1113 
1114  }
1115  H5Tclose(item_type0);
1116  H5Tclose(fieldtype0);
1117  break;
1118 
1119  }
1120  case UUID: {
1121  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
1122  unsigned int total_size0=H5Tget_size(fieldtype0);
1123  boost::uuids::uuid x0;
1124  memcpy(&x0,buf+offset,total_size0);
1125  is_row_selected=CmpConds<boost::uuids::uuid>(&x0,&(field_conds[qr.fields[j]]));
1126  if(is_row_selected){
1127  row[j]=x0;
1128 
1129  }
1130  H5Tclose(fieldtype0);
1131  break;
1132 
1133  }
1135  std::map<std::pair<int, std::string>, double> x0;
1136  x0=VLRead<std::map<std::pair<int, std::string>, double>,VL_MAP_PAIR_INT_STRING_DOUBLE>(buf+offset);
1137  is_row_selected=CmpConds<std::map<std::pair<int, std::string>, double>>(&x0,&(field_conds[qr.fields[j]]));
1138  if(is_row_selected){
1139  row[j]=x0;
1140 
1141  }
1142  break;
1143 
1144  }
1145  case INT: {
1146  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
1147  unsigned int total_size0=H5Tget_size(fieldtype0);
1148  int x0;
1149  x0=*reinterpret_cast<int*>(buf+offset);
1150  is_row_selected=CmpConds<int>(&x0,&(field_conds[qr.fields[j]]));
1151  if(is_row_selected){
1152  row[j]=x0;
1153 
1154  }
1155  H5Tclose(fieldtype0);
1156  break;
1157 
1158  }
1159  case DOUBLE: {
1160  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
1161  unsigned int total_size0=H5Tget_size(fieldtype0);
1162  double x0;
1163  x0=*reinterpret_cast<double*>(buf+offset);
1164  is_row_selected=CmpConds<double>(&x0,&(field_conds[qr.fields[j]]));
1165  if(is_row_selected){
1166  row[j]=x0;
1167 
1168  }
1169  H5Tclose(fieldtype0);
1170  break;
1171 
1172  }
1173  case FLOAT: {
1174  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
1175  unsigned int total_size0=H5Tget_size(fieldtype0);
1176  float x0;
1177  x0=*reinterpret_cast<float*>(buf+offset);
1178  is_row_selected=CmpConds<float>(&x0,&(field_conds[qr.fields[j]]));
1179  if(is_row_selected){
1180  row[j]=x0;
1181 
1182  }
1183  H5Tclose(fieldtype0);
1184  break;
1185 
1186  }
1187  case MAP_INT_DOUBLE: {
1188  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
1189  unsigned int total_size0=H5Tget_size(fieldtype0);
1190  hsize_t fieldlen0;
1191  H5Tget_array_dims2(fieldtype0,&fieldlen0);
1192  hid_t item_type0=H5Tget_super(fieldtype0);
1193  hid_t fieldtype1key=H5Tget_member_type(item_type0,0);
1194  unsigned int total_size1key=H5Tget_size(fieldtype1key);
1195  hid_t fieldtype1val=H5Tget_member_type(item_type0,1);
1196  unsigned int total_size1val=H5Tget_size(fieldtype1val);
1197  std::map<int, double> x0;
1198  for(unsigned int k0=0;k0<fieldlen0;++k0){
1199  int x1key;
1200  x1key=*reinterpret_cast<int*>(buf+offset+(total_size1key+total_size1val)*k0);
1201  double x1val;
1202  x1val=*reinterpret_cast<double*>(buf+offset+(total_size1key+total_size1val)*k0+total_size1key);
1203  x0[x1key]=x1val;
1204 
1205  }
1206  is_row_selected=CmpConds<std::map<int, double>>(&x0,&(field_conds[qr.fields[j]]));
1207  if(is_row_selected){
1208  row[j]=x0;
1209 
1210  }
1211  H5Tclose(fieldtype1val);
1212  H5Tclose(fieldtype1key);
1213  H5Tclose(item_type0);
1214  H5Tclose(fieldtype0);
1215  break;
1216 
1217  }
1218  case SET_INT: {
1219  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
1220  unsigned int total_size0=H5Tget_size(fieldtype0);
1221  hsize_t fieldlen0;
1222  H5Tget_array_dims2(fieldtype0,&fieldlen0);
1223  hid_t item_type0=H5Tget_super(fieldtype0);
1224  unsigned int total_size1elem=H5Tget_size(item_type0);
1225  std::set<int> x0;
1226  int* xraw1elem=reinterpret_cast<int*>(buf+offset);
1227  x0=std::set<int>(xraw1elem,xraw1elem+fieldlen0);
1228  is_row_selected=CmpConds<std::set<int>>(&x0,&(field_conds[qr.fields[j]]));
1229  if(is_row_selected){
1230  row[j]=x0;
1231 
1232  }
1233  H5Tclose(item_type0);
1234  H5Tclose(fieldtype0);
1235  break;
1236 
1237  }
1238  case MAP_INT_INT: {
1239  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
1240  unsigned int total_size0=H5Tget_size(fieldtype0);
1241  hsize_t fieldlen0;
1242  H5Tget_array_dims2(fieldtype0,&fieldlen0);
1243  hid_t item_type0=H5Tget_super(fieldtype0);
1244  hid_t fieldtype1key=H5Tget_member_type(item_type0,0);
1245  unsigned int total_size1key=H5Tget_size(fieldtype1key);
1246  hid_t fieldtype1val=H5Tget_member_type(item_type0,1);
1247  unsigned int total_size1val=H5Tget_size(fieldtype1val);
1248  std::map<int, int> x0;
1249  for(unsigned int k0=0;k0<fieldlen0;++k0){
1250  int x1key;
1251  x1key=*reinterpret_cast<int*>(buf+offset+(total_size1key+total_size1val)*k0);
1252  int x1val;
1253  x1val=*reinterpret_cast<int*>(buf+offset+(total_size1key+total_size1val)*k0+total_size1key);
1254  x0[x1key]=x1val;
1255 
1256  }
1257  is_row_selected=CmpConds<std::map<int, int>>(&x0,&(field_conds[qr.fields[j]]));
1258  if(is_row_selected){
1259  row[j]=x0;
1260 
1261  }
1262  H5Tclose(fieldtype1val);
1263  H5Tclose(fieldtype1key);
1264  H5Tclose(item_type0);
1265  H5Tclose(fieldtype0);
1266  break;
1267 
1268  }
1269  case VL_LIST_STRING: {
1270  std::list<std::string> x0;
1271  x0=VLRead<std::list<std::string>,VL_LIST_STRING>(buf+offset);
1272  is_row_selected=CmpConds<std::list<std::string>>(&x0,&(field_conds[qr.fields[j]]));
1273  if(is_row_selected){
1274  row[j]=x0;
1275 
1276  }
1277  break;
1278 
1279  }
1280  case VL_VECTOR_VL_STRING: {
1281  std::vector<std::string> x0;
1282  x0=VLRead<std::vector<std::string>,VL_VECTOR_VL_STRING>(buf+offset);
1283  is_row_selected=CmpConds<std::vector<std::string>>(&x0,&(field_conds[qr.fields[j]]));
1284  if(is_row_selected){
1285  row[j]=x0;
1286 
1287  }
1288  break;
1289 
1290  }
1291  case BOOL: {
1292  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
1293  unsigned int total_size0=H5Tget_size(fieldtype0);
1294  bool x0;
1295  x0=*reinterpret_cast<bool*>(buf+offset);
1296  is_row_selected=CmpConds<bool>(&x0,&(field_conds[qr.fields[j]]));
1297  if(is_row_selected){
1298  row[j]=x0;
1299 
1300  }
1301  H5Tclose(fieldtype0);
1302  break;
1303 
1304  }
1305  case BLOB: {
1306  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
1307  unsigned int total_size0=H5Tget_size(fieldtype0);
1308  cyclus::Blob x0;
1309  x0=VLRead<cyclus::Blob,BLOB>(buf+offset);
1310  is_row_selected=CmpConds<cyclus::Blob>(&x0,&(field_conds[qr.fields[j]]));
1311  if(is_row_selected){
1312  row[j]=x0;
1313 
1314  }
1315  H5Tclose(fieldtype0);
1316  break;
1317 
1318  }
1319  case VL_MAP_VL_STRING_DOUBLE: {
1320  std::map<std::string, double> x0;
1321  x0=VLRead<std::map<std::string, double>,VL_MAP_VL_STRING_DOUBLE>(buf+offset);
1322  is_row_selected=CmpConds<std::map<std::string, double>>(&x0,&(field_conds[qr.fields[j]]));
1323  if(is_row_selected){
1324  row[j]=x0;
1325 
1326  }
1327  break;
1328 
1329  }
1330  case VECTOR_VL_STRING: {
1331  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
1332  unsigned int total_size0=H5Tget_size(fieldtype0);
1333  hsize_t fieldlen0;
1334  H5Tget_array_dims2(fieldtype0,&fieldlen0);
1335  hid_t item_type0=H5Tget_super(fieldtype0);
1336  unsigned int total_size1elem=H5Tget_size(item_type0);
1337  std::vector<std::string> x0;
1338  x0=std::vector<std::string>(fieldlen0);
1339  for(unsigned int k0=0;k0<fieldlen0;++k0){
1340  std::string x1elem;
1341  x1elem=VLRead<std::string,VL_STRING>(buf+offset+total_size1elem*k0);
1342  x0[k0]=x1elem;
1343 
1344  }
1345  is_row_selected=CmpConds<std::vector<std::string>>(&x0,&(field_conds[qr.fields[j]]));
1346  if(is_row_selected){
1347  row[j]=x0;
1348 
1349  }
1350  H5Tclose(item_type0);
1351  H5Tclose(fieldtype0);
1352  break;
1353 
1354  }
1356  std::map<std::string, std::string> x0;
1357  x0=VLRead<std::map<std::string, std::string>,VL_MAP_VL_STRING_VL_STRING>(buf+offset);
1358  is_row_selected=CmpConds<std::map<std::string, std::string>>(&x0,&(field_conds[qr.fields[j]]));
1359  if(is_row_selected){
1360  row[j]=x0;
1361 
1362  }
1363  break;
1364 
1365  }
1366  case VL_VECTOR_DOUBLE: {
1367  std::vector<double> x0;
1368  x0=VLRead<std::vector<double>,VL_VECTOR_DOUBLE>(buf+offset);
1369  is_row_selected=CmpConds<std::vector<double>>(&x0,&(field_conds[qr.fields[j]]));
1370  if(is_row_selected){
1371  row[j]=x0;
1372 
1373  }
1374  break;
1375 
1376  }
1377  case MAP_STRING_STRING: {
1378  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
1379  unsigned int total_size0=H5Tget_size(fieldtype0);
1380  hsize_t fieldlen0;
1381  H5Tget_array_dims2(fieldtype0,&fieldlen0);
1382  hid_t item_type0=H5Tget_super(fieldtype0);
1383  size_t nullpos1key;
1384  hid_t fieldtype1key=H5Tget_member_type(item_type0,0);
1385  unsigned int total_size1key=H5Tget_size(fieldtype1key);
1386  size_t nullpos1val;
1387  hid_t fieldtype1val=H5Tget_member_type(item_type0,1);
1388  unsigned int total_size1val=H5Tget_size(fieldtype1val);
1389  std::map<std::string, std::string> x0;
1390  for(unsigned int k0=0;k0<fieldlen0;++k0){
1391  std::string x1key;
1392  x1key=std::string(buf+offset+(total_size1key+total_size1val)*k0,total_size1key);
1393  nullpos1key=x1key.find('\0');
1394  if(nullpos1key!=std::string::npos){
1395  x1key.resize(nullpos1key);
1396 
1397  }
1398  std::string x1val;
1399  x1val=std::string(buf+offset+(total_size1key+total_size1val)*k0+total_size1key,total_size1val);
1400  nullpos1val=x1val.find('\0');
1401  if(nullpos1val!=std::string::npos){
1402  x1val.resize(nullpos1val);
1403 
1404  }
1405  x0[x1key]=x1val;
1406 
1407  }
1408  is_row_selected=CmpConds<std::map<std::string, std::string>>(&x0,&(field_conds[qr.fields[j]]));
1409  if(is_row_selected){
1410  row[j]=x0;
1411 
1412  }
1413  H5Tclose(fieldtype1val);
1414  H5Tclose(fieldtype1key);
1415  H5Tclose(item_type0);
1416  H5Tclose(fieldtype0);
1417  break;
1418 
1419  }
1420  case VL_VECTOR_FLOAT: {
1421  std::vector<float> x0;
1422  x0=VLRead<std::vector<float>,VL_VECTOR_FLOAT>(buf+offset);
1423  is_row_selected=CmpConds<std::vector<float>>(&x0,&(field_conds[qr.fields[j]]));
1424  if(is_row_selected){
1425  row[j]=x0;
1426 
1427  }
1428  break;
1429 
1430  }
1431  case VL_MAP_VL_STRING_INT: {
1432  std::map<std::string, int> x0;
1433  x0=VLRead<std::map<std::string, int>,VL_MAP_VL_STRING_INT>(buf+offset);
1434  is_row_selected=CmpConds<std::map<std::string, int>>(&x0,&(field_conds[qr.fields[j]]));
1435  if(is_row_selected){
1436  row[j]=x0;
1437 
1438  }
1439  break;
1440 
1441  }
1442  case LIST_INT: {
1443  hid_t fieldtype0=H5Tget_member_type(tb_type,j);
1444  unsigned int total_size0=H5Tget_size(fieldtype0);
1445  hsize_t fieldlen0;
1446  H5Tget_array_dims2(fieldtype0,&fieldlen0);
1447  hid_t item_type0=H5Tget_super(fieldtype0);
1448  unsigned int total_size1elem=H5Tget_size(item_type0);
1449  std::list<int> x0;
1450  int* xraw1elem=reinterpret_cast<int*>(buf+offset);
1451  x0=std::list<int>(xraw1elem,xraw1elem+fieldlen0);
1452  is_row_selected=CmpConds<std::list<int>>(&x0,&(field_conds[qr.fields[j]]));
1453  if(is_row_selected){
1454  row[j]=x0;
1455 
1456  }
1457  H5Tclose(item_type0);
1458  H5Tclose(fieldtype0);
1459  break;
1460 
1461  }
1462 
1463 
1464  default: {
1465  throw IOError("querying column '" + qr.fields[j] + "' in table '" + \
1466  table + "' failed due to unsupported data type.");
1467  break;
1468  }
1469  }
1470  if (!is_row_selected)
1471  break;
1472  offset += col_sizes_[table][j];
1473  }
1474  if (is_row_selected) {
1475  qr.rows.push_back(row);
1476  }
1477  }
1478  delete[] buf;
1479  H5Sclose(memspace);
1480  }
1481 
1482  // close and return
1483  H5Tclose(tb_type);
1484  H5Pclose(tb_plist);
1485  H5Sclose(tb_space);
1486  H5Dclose(tb_set);
1487  return qr;
1488 }
1489 
1490 QueryResult Hdf5Back::GetTableInfo(std::string title, hid_t dset, hid_t dt) {
1491  int i;
1492  char * colname;
1493  hsize_t ncols = H5Tget_nmembers(dt);
1494  std::string fieldname;
1495  std::string fieldtype;
1496  LoadTableTypes(title, dset, ncols);
1497  DbTypes* dbtypes = schemas_[title];
1498 
1499  QueryResult qr;
1500  for (i = 0; i < ncols; ++i) {
1501  colname = H5Tget_member_name(dt, i);
1502  fieldname = std::string(colname);
1503  free(colname);
1504  qr.fields.push_back(fieldname);
1505  qr.types.push_back(dbtypes[i]);
1506  }
1507  return qr;
1508 }
1509 
1510 void Hdf5Back::LoadTableTypes(std::string title, hsize_t ncols) {
1511  if (schemas_.count(title) > 0)
1512  return;
1513  hid_t dset = H5Dopen2(file_, title.c_str(), H5P_DEFAULT);
1514  LoadTableTypes(title, dset, ncols);
1515  H5Dclose(dset);
1516 }
1517 
1518 void Hdf5Back::LoadTableTypes(std::string title, hid_t dset, hsize_t ncols) {
1519  if (schemas_.count(title) > 0)
1520  return;
1521 
1522  int i;
1523  hid_t subt;
1524  hid_t t = H5Dget_type(dset);
1525  schema_sizes_[title] = H5Tget_size(t);
1526  size_t* offsets = new size_t[ncols];
1527  size_t* sizes = new size_t[ncols];
1528  for (i = 0; i < ncols; ++i) {
1529  offsets[i] = H5Tget_member_offset(t, i);
1530  subt = H5Tget_member_type(t, i);
1531  sizes[i] = H5Tget_size(subt);
1532  H5Tclose(subt);
1533  }
1534  H5Tclose(t);
1535  col_offsets_[title] = offsets;
1536  col_sizes_[title] = sizes;
1537 
1538  // get types from db
1539  int dbt[ncols];
1540  hid_t dbtypes_attr = H5Aopen(dset, "cyclus_dbtypes", H5P_DEFAULT);
1541  hid_t dbtypes_type = H5Aget_type(dbtypes_attr);
1542  H5Aread(dbtypes_attr, dbtypes_type, dbt);
1543  H5Tclose(dbtypes_type);
1544  H5Aclose(dbtypes_attr);
1545 
1546  // store types
1547  DbTypes* dbtypes = new DbTypes[ncols];
1548  for (i = 0; i < ncols; ++i)
1549  dbtypes[i] = static_cast<DbTypes>(dbt[i]);
1550  schemas_[title] = dbtypes;
1551 }
1552 
1553 hid_t Hdf5Back::CreateFLStrType(int n) {
1554  hid_t str_type = H5Tcopy(H5T_C_S1);
1555  H5Tset_size(str_type, n);
1556  H5Tset_strpad(str_type, H5T_STR_NULLPAD);
1557  opened_types_.insert(str_type);
1558  return str_type;
1559 }
1560 
1561 std::string Hdf5Back::Name() {
1562  return path_;
1563 }
1564 
1565 void Hdf5Back::CreateTable(Datum* d) {
1566  using std::set;
1567  using std::string;
1568  using std::vector;
1569  using std::list;
1570  using std::pair;
1571  using std::list;
1572  using std::map;
1573  Datum::Vals vals = d->vals();
1574  hsize_t nvals = vals.size();
1575  Datum::Shape shape;
1576  Datum::Shapes shapes = d->shapes();
1577 
1578  herr_t status;
1579  size_t dst_size = 0;
1580  size_t* dst_offset = new size_t[nvals];
1581  size_t* dst_sizes = new size_t[nvals];
1582  hid_t field_types[nvals];
1583  DbTypes* dbtypes = new DbTypes[nvals];
1584  const char* field_names[nvals];
1585  for (int i = 0; i < nvals; ++i) {
1586  dst_offset[i] = dst_size;
1587  field_names[i] = vals[i].first;
1588  const std::type_info& valtype = vals[i].second.type();
1589  if(valtype==typeid(std::vector<int>)){
1590  shape=shapes[i];
1591  if(shape.empty()||shape[0]<1){
1592  dbtypes[i]=VL_VECTOR_INT;
1593  hid_t item_type0;
1594  hid_t item_type1elem;
1595  item_type1elem=H5T_NATIVE_INT;
1596  dst_sizes[i]=(CYCLUS_SHA1_SIZE);
1597  field_types[i]=sha1_type_;
1598  if(vldts_.count(VL_VECTOR_INT)==0){
1599  vldts_[VL_VECTOR_INT]=H5Tvlen_create(item_type1elem);
1600  opened_types_.insert(vldts_[VL_VECTOR_INT]);
1601 
1602  }
1603 
1604  }else{
1605  dbtypes[i]=VECTOR_INT;
1606  hid_t item_type0;
1607  hsize_t shape01=shape[0];
1608  hid_t item_type1elem;
1609  item_type1elem=H5T_NATIVE_INT;
1610  item_type0=H5Tarray_create2(item_type1elem,1,&shape01);
1611  dst_sizes[i]=((sizeof(int))*shape[0]);
1612  field_types[i]=item_type0;
1613  opened_types_.insert(field_types[i]);
1614 
1615  }
1616 
1617  }else if(valtype==typeid(std::vector<std::string>)){
1618  shape=shapes[i];
1619  if(shape[0]>=1&&shape[1]<1){
1620  dbtypes[i]=VECTOR_VL_STRING;
1621  hid_t item_type0;
1622  hsize_t shape01=shape[0];
1623  hid_t item_type1elem;
1624  item_type1elem=sha1_type_;
1625  item_type0=H5Tarray_create2(item_type1elem,1,&shape01);
1626  dst_sizes[i]=((CYCLUS_SHA1_SIZE)*shape[0]);
1627  field_types[i]=item_type0;
1628  opened_types_.insert(field_types[i]);
1629 
1630  }else if(shape[0]<1&&shape[1]>=1){
1631  dbtypes[i]=VL_VECTOR_STRING;
1632  hid_t item_type0;
1633  hid_t item_type1elem;
1634  item_type1elem=sha1_type_;
1635  dst_sizes[i]=(CYCLUS_SHA1_SIZE);
1636  field_types[i]=sha1_type_;
1637  if(vldts_.count(VL_VECTOR_STRING)==0){
1638  vldts_[VL_VECTOR_STRING]=H5Tvlen_create(item_type1elem);
1639  opened_types_.insert(vldts_[VL_VECTOR_STRING]);
1640 
1641  }
1642 
1643  }else if(shape.empty()||shape[0]<1&&shape[1]<1){
1644  dbtypes[i]=VL_VECTOR_VL_STRING;
1645  hid_t item_type0;
1646  hid_t item_type1elem;
1647  item_type1elem=sha1_type_;
1648  dst_sizes[i]=(CYCLUS_SHA1_SIZE);
1649  field_types[i]=sha1_type_;
1650  if(vldts_.count(VL_VECTOR_VL_STRING)==0){
1651  vldts_[VL_VECTOR_VL_STRING]=H5Tvlen_create(item_type1elem);
1652  opened_types_.insert(vldts_[VL_VECTOR_VL_STRING]);
1653 
1654  }
1655 
1656  }else{
1657  dbtypes[i]=VECTOR_STRING;
1658  hid_t item_type0;
1659  hsize_t shape01=shape[0];
1660  hid_t item_type1elem;
1661  item_type1elem=CreateFLStrType(shape[1]);
1662  item_type0=H5Tarray_create2(item_type1elem,1,&shape01);
1663  dst_sizes[i]=((shape[1])*shape[0]);
1664  field_types[i]=item_type0;
1665  opened_types_.insert(field_types[i]);
1666 
1667  }
1668 
1669  }else if(valtype==typeid(std::pair<int, std::string>)){
1670  shape=shapes[i];
1671  if(shape.empty()||shape[2]<1){
1672  dbtypes[i]=PAIR_INT_VL_STRING;
1673  hid_t item_type0;
1674  hid_t item_type1first;
1675  item_type1first=H5T_NATIVE_INT;
1676  hid_t item_type1second;
1677  item_type1second=sha1_type_;
1678  hid_t item_type1;
1679  item_type1=H5Tcreate(H5T_COMPOUND,sizeof(int)+CYCLUS_SHA1_SIZE);
1680  H5Tinsert(item_type1,"first",0,item_type1first);
1681  H5Tinsert(item_type1,"second",0+sizeof(int),item_type1second);
1682  dst_sizes[i]=((sizeof(int)+CYCLUS_SHA1_SIZE));
1683  field_types[i]=item_type1;
1684  opened_types_.insert(field_types[i]);
1685 
1686  }else{
1687  dbtypes[i]=PAIR_INT_STRING;
1688  hid_t item_type0;
1689  hid_t item_type1first;
1690  item_type1first=H5T_NATIVE_INT;
1691  hid_t item_type1second;
1692  item_type1second=CreateFLStrType(shape[2]);
1693  hid_t item_type1;
1694  item_type1=H5Tcreate(H5T_COMPOUND,sizeof(int)+shape[2]);
1695  H5Tinsert(item_type1,"first",0,item_type1first);
1696  H5Tinsert(item_type1,"second",0+sizeof(int),item_type1second);
1697  dst_sizes[i]=((sizeof(int)+shape[2]));
1698  field_types[i]=item_type1;
1699  opened_types_.insert(field_types[i]);
1700 
1701  }
1702 
1703  }else if(valtype==typeid(std::vector<float>)){
1704  shape=shapes[i];
1705  if(shape.empty()||shape[0]<1){
1706  dbtypes[i]=VL_VECTOR_FLOAT;
1707  hid_t item_type0;
1708  hid_t item_type1elem;
1709  item_type1elem=H5T_NATIVE_FLOAT;
1710  dst_sizes[i]=(CYCLUS_SHA1_SIZE);
1711  field_types[i]=sha1_type_;
1712  if(vldts_.count(VL_VECTOR_FLOAT)==0){
1713  vldts_[VL_VECTOR_FLOAT]=H5Tvlen_create(item_type1elem);
1714  opened_types_.insert(vldts_[VL_VECTOR_FLOAT]);
1715 
1716  }
1717 
1718  }else{
1719  dbtypes[i]=VECTOR_FLOAT;
1720  hid_t item_type0;
1721  hsize_t shape01=shape[0];
1722  hid_t item_type1elem;
1723  item_type1elem=H5T_NATIVE_FLOAT;
1724  item_type0=H5Tarray_create2(item_type1elem,1,&shape01);
1725  dst_sizes[i]=((sizeof(float))*shape[0]);
1726  field_types[i]=item_type0;
1727  opened_types_.insert(field_types[i]);
1728 
1729  }
1730 
1731  }else if(valtype==typeid(std::string)){
1732  shape=shapes[i];
1733  if(shape.empty()||shape[0]<1){
1734  dbtypes[i]=VL_STRING;
1735  hid_t item_type0;
1736  item_type0=sha1_type_;
1737  dst_sizes[i]=CYCLUS_SHA1_SIZE;
1738  field_types[i]=sha1_type_;
1739  if(vldts_.count(VL_STRING)==0){
1740  vldts_[VL_STRING]=H5Tvlen_create(item_type0);
1741  opened_types_.insert(vldts_[VL_STRING]);
1742 
1743  }
1744 
1745  }else{
1746  dbtypes[i]=STRING;
1747  field_types[i]=H5Tcopy(H5T_C_S1);
1748  H5Tset_size(field_types[i], shape[0]);
1749  H5Tset_strpad(field_types[i], H5T_STR_NULLPAD);
1750  opened_types_.insert(field_types[i]);
1751  dst_sizes[i]=sizeof(char)*shape[0];
1752 
1753  }
1754 
1755  }else if(valtype==typeid(std::list<std::string>)){
1756  shape=shapes[i];
1757  if(shape[0]<1&&shape[1]>=1){
1758  dbtypes[i]=VL_LIST_STRING;
1759  hid_t item_type0;
1760  hid_t item_type1elem;
1761  item_type1elem=sha1_type_;
1762  dst_sizes[i]=(CYCLUS_SHA1_SIZE);
1763  field_types[i]=sha1_type_;
1764  if(vldts_.count(VL_LIST_STRING)==0){
1765  vldts_[VL_LIST_STRING]=H5Tvlen_create(item_type1elem);
1766  opened_types_.insert(vldts_[VL_LIST_STRING]);
1767 
1768  }
1769 
1770  }else if(shape[0]>=1&&shape[1]<1){
1771  dbtypes[i]=LIST_VL_STRING;
1772  hid_t item_type0;
1773  hsize_t shape01=shape[0];
1774  hid_t item_type1elem;
1775  item_type1elem=sha1_type_;
1776  item_type0=H5Tarray_create2(item_type1elem,1,&shape01);
1777  dst_sizes[i]=((CYCLUS_SHA1_SIZE)*shape[0]);
1778  field_types[i]=item_type0;
1779  opened_types_.insert(field_types[i]);
1780 
1781  }else if(shape.empty()||shape[0]<1&&shape[1]<1){
1782  dbtypes[i]=VL_LIST_VL_STRING;
1783  hid_t item_type0;
1784  hid_t item_type1elem;
1785  item_type1elem=sha1_type_;
1786  dst_sizes[i]=(CYCLUS_SHA1_SIZE);
1787  field_types[i]=sha1_type_;
1788  if(vldts_.count(VL_LIST_VL_STRING)==0){
1789  vldts_[VL_LIST_VL_STRING]=H5Tvlen_create(item_type1elem);
1790  opened_types_.insert(vldts_[VL_LIST_VL_STRING]);
1791 
1792  }
1793 
1794  }else{
1795  dbtypes[i]=LIST_STRING;
1796  hid_t item_type0;
1797  hsize_t shape01=shape[0];
1798  hid_t item_type1elem;
1799  item_type1elem=CreateFLStrType(shape[1]);
1800  item_type0=H5Tarray_create2(item_type1elem,1,&shape01);
1801  dst_sizes[i]=((shape[1])*shape[0]);
1802  field_types[i]=item_type0;
1803  opened_types_.insert(field_types[i]);
1804 
1805  }
1806 
1807  }else if(valtype==typeid(std::map<std::string, double>)){
1808  shape=shapes[i];
1809  if(shape.empty()||shape[0]<1&&shape[1]<1){
1810  dbtypes[i]=VL_MAP_VL_STRING_DOUBLE;
1811  hid_t item_type0;
1812  hid_t item_type1key;
1813  item_type1key=sha1_type_;
1814  hid_t item_type1val;
1815  item_type1val=H5T_NATIVE_DOUBLE;
1816  hid_t item_type1;
1817  item_type1=H5Tcreate(H5T_COMPOUND,CYCLUS_SHA1_SIZE+sizeof(double));
1818  H5Tinsert(item_type1,"key",0,item_type1key);
1819  H5Tinsert(item_type1,"val",0+CYCLUS_SHA1_SIZE,item_type1val);
1820  dst_sizes[i]=(CYCLUS_SHA1_SIZE);
1821  field_types[i]=sha1_type_;
1822  if(vldts_.count(VL_MAP_VL_STRING_DOUBLE)==0){
1823  vldts_[VL_MAP_VL_STRING_DOUBLE]=H5Tvlen_create(item_type1);
1824  opened_types_.insert(vldts_[VL_MAP_VL_STRING_DOUBLE]);
1825 
1826  }
1827 
1828  }else if(shape[0]<1&&shape[1]>=1){
1829  dbtypes[i]=VL_MAP_STRING_DOUBLE;
1830  hid_t item_type0;
1831  hid_t item_type1key;
1832  item_type1key=sha1_type_;
1833  hid_t item_type1val;
1834  item_type1val=H5T_NATIVE_DOUBLE;
1835  hid_t item_type1;
1836  item_type1=H5Tcreate(H5T_COMPOUND,CYCLUS_SHA1_SIZE+sizeof(double));
1837  H5Tinsert(item_type1,"key",0,item_type1key);
1838  H5Tinsert(item_type1,"val",0+CYCLUS_SHA1_SIZE,item_type1val);
1839  dst_sizes[i]=(CYCLUS_SHA1_SIZE);
1840  field_types[i]=sha1_type_;
1841  if(vldts_.count(VL_MAP_STRING_DOUBLE)==0){
1842  vldts_[VL_MAP_STRING_DOUBLE]=H5Tvlen_create(item_type1);
1843  opened_types_.insert(vldts_[VL_MAP_STRING_DOUBLE]);
1844 
1845  }
1846 
1847  }else if(shape[0]>=1&&shape[1]<1){
1848  dbtypes[i]=MAP_VL_STRING_DOUBLE;
1849  hid_t item_type0;
1850  hsize_t shape01=shape[0];
1851  hid_t item_type1key;
1852  item_type1key=sha1_type_;
1853  hid_t item_type1val;
1854  item_type1val=H5T_NATIVE_DOUBLE;
1855  hid_t item_type1;
1856  item_type1=H5Tcreate(H5T_COMPOUND,CYCLUS_SHA1_SIZE+sizeof(double));
1857  H5Tinsert(item_type1,"key",0,item_type1key);
1858  H5Tinsert(item_type1,"val",0+CYCLUS_SHA1_SIZE,item_type1val);
1859  item_type0=H5Tarray_create2(item_type1,1,&shape01);
1860  dst_sizes[i]=((CYCLUS_SHA1_SIZE+sizeof(double))*shape[0]);
1861  field_types[i]=item_type0;
1862  opened_types_.insert(item_type1);
1863  opened_types_.insert(field_types[i]);
1864 
1865  }else{
1866  dbtypes[i]=MAP_STRING_DOUBLE;
1867  hid_t item_type0;
1868  hsize_t shape01=shape[0];
1869  hid_t item_type1key;
1870  item_type1key=CreateFLStrType(shape[1]);
1871  hid_t item_type1val;
1872  item_type1val=H5T_NATIVE_DOUBLE;
1873  hid_t item_type1;
1874  item_type1=H5Tcreate(H5T_COMPOUND,shape[1]+sizeof(double));
1875  H5Tinsert(item_type1,"key",0,item_type1key);
1876  H5Tinsert(item_type1,"val",0+shape[1],item_type1val);
1877  item_type0=H5Tarray_create2(item_type1,1,&shape01);
1878  dst_sizes[i]=((shape[1]+sizeof(double))*shape[0]);
1879  field_types[i]=item_type0;
1880  opened_types_.insert(item_type1);
1881  opened_types_.insert(field_types[i]);
1882 
1883  }
1884 
1885  }else if(valtype==typeid(std::set<std::string>)){
1886  shape=shapes[i];
1887  if(shape[0]>=1&&shape[1]<1){
1888  dbtypes[i]=SET_VL_STRING;
1889  hid_t item_type0;
1890  hsize_t shape01=shape[0];
1891  hid_t item_type1elem;
1892  item_type1elem=sha1_type_;
1893  item_type0=H5Tarray_create2(item_type1elem,1,&shape01);
1894  dst_sizes[i]=((CYCLUS_SHA1_SIZE)*shape[0]);
1895  field_types[i]=item_type0;
1896  opened_types_.insert(field_types[i]);
1897 
1898  }else if(shape[0]<1&&shape[1]>=1){
1899  dbtypes[i]=VL_SET_STRING;
1900  hid_t item_type0;
1901  hid_t item_type1elem;
1902  item_type1elem=sha1_type_;
1903  dst_sizes[i]=(CYCLUS_SHA1_SIZE);
1904  field_types[i]=sha1_type_;
1905  if(vldts_.count(VL_SET_STRING)==0){
1906  vldts_[VL_SET_STRING]=H5Tvlen_create(item_type1elem);
1907  opened_types_.insert(vldts_[VL_SET_STRING]);
1908 
1909  }
1910 
1911  }else if(shape.empty()||shape[0]<1&&shape[1]<1){
1912  dbtypes[i]=VL_SET_VL_STRING;
1913  hid_t item_type0;
1914  hid_t item_type1elem;
1915  item_type1elem=sha1_type_;
1916  dst_sizes[i]=(CYCLUS_SHA1_SIZE);
1917  field_types[i]=sha1_type_;
1918  if(vldts_.count(VL_SET_VL_STRING)==0){
1919  vldts_[VL_SET_VL_STRING]=H5Tvlen_create(item_type1elem);
1920  opened_types_.insert(vldts_[VL_SET_VL_STRING]);
1921 
1922  }
1923 
1924  }else{
1925  dbtypes[i]=SET_STRING;
1926  hid_t item_type0;
1927  hsize_t shape01=shape[0];
1928  hid_t item_type1elem;
1929  item_type1elem=CreateFLStrType(shape[1]);
1930  item_type0=H5Tarray_create2(item_type1elem,1,&shape01);
1931  dst_sizes[i]=((shape[1])*shape[0]);
1932  field_types[i]=item_type0;
1933  opened_types_.insert(field_types[i]);
1934 
1935  }
1936 
1937  }else if(valtype==typeid(std::map<std::string, int>)){
1938  shape=shapes[i];
1939  if(shape.empty()||shape[0]<1&&shape[1]<1){
1940  dbtypes[i]=VL_MAP_VL_STRING_INT;
1941  hid_t item_type0;
1942  hid_t item_type1key;
1943  item_type1key=sha1_type_;
1944  hid_t item_type1val;
1945  item_type1val=H5T_NATIVE_INT;
1946  hid_t item_type1;
1947  item_type1=H5Tcreate(H5T_COMPOUND,CYCLUS_SHA1_SIZE+sizeof(int));
1948  H5Tinsert(item_type1,"key",0,item_type1key);
1949  H5Tinsert(item_type1,"val",0+CYCLUS_SHA1_SIZE,item_type1val);
1950  dst_sizes[i]=(CYCLUS_SHA1_SIZE);
1951  field_types[i]=sha1_type_;
1952  if(vldts_.count(VL_MAP_VL_STRING_INT)==0){
1953  vldts_[VL_MAP_VL_STRING_INT]=H5Tvlen_create(item_type1);
1954  opened_types_.insert(vldts_[VL_MAP_VL_STRING_INT]);
1955 
1956  }
1957 
1958  }else if(shape[0]>=1&&shape[1]<1){
1959  dbtypes[i]=MAP_VL_STRING_INT;
1960  hid_t item_type0;
1961  hsize_t shape01=shape[0];
1962  hid_t item_type1key;
1963  item_type1key=sha1_type_;
1964  hid_t item_type1val;
1965  item_type1val=H5T_NATIVE_INT;
1966  hid_t item_type1;
1967  item_type1=H5Tcreate(H5T_COMPOUND,CYCLUS_SHA1_SIZE+sizeof(int));
1968  H5Tinsert(item_type1,"key",0,item_type1key);
1969  H5Tinsert(item_type1,"val",0+CYCLUS_SHA1_SIZE,item_type1val);
1970  item_type0=H5Tarray_create2(item_type1,1,&shape01);
1971  dst_sizes[i]=((CYCLUS_SHA1_SIZE+sizeof(int))*shape[0]);
1972  field_types[i]=item_type0;
1973  opened_types_.insert(item_type1);
1974  opened_types_.insert(field_types[i]);
1975 
1976  }else if(shape[0]<1&&shape[1]>=1){
1977  dbtypes[i]=VL_MAP_STRING_INT;
1978  hid_t item_type0;
1979  hid_t item_type1key;
1980  item_type1key=sha1_type_;
1981  hid_t item_type1val;
1982  item_type1val=H5T_NATIVE_INT;
1983  hid_t item_type1;
1984  item_type1=H5Tcreate(H5T_COMPOUND,CYCLUS_SHA1_SIZE+sizeof(int));
1985  H5Tinsert(item_type1,"key",0,item_type1key);
1986  H5Tinsert(item_type1,"val",0+CYCLUS_SHA1_SIZE,item_type1val);
1987  dst_sizes[i]=(CYCLUS_SHA1_SIZE);
1988  field_types[i]=sha1_type_;
1989  if(vldts_.count(VL_MAP_STRING_INT)==0){
1990  vldts_[VL_MAP_STRING_INT]=H5Tvlen_create(item_type1);
1991  opened_types_.insert(vldts_[VL_MAP_STRING_INT]);
1992 
1993  }
1994 
1995  }else{
1996  dbtypes[i]=MAP_STRING_INT;
1997  hid_t item_type0;
1998  hsize_t shape01=shape[0];
1999  hid_t item_type1key;
2000  item_type1key=CreateFLStrType(shape[1]);
2001  hid_t item_type1val;
2002  item_type1val=H5T_NATIVE_INT;
2003  hid_t item_type1;
2004  item_type1=H5Tcreate(H5T_COMPOUND,shape[1]+sizeof(int));
2005  H5Tinsert(item_type1,"key",0,item_type1key);
2006  H5Tinsert(item_type1,"val",0+shape[1],item_type1val);
2007  item_type0=H5Tarray_create2(item_type1,1,&shape01);
2008  dst_sizes[i]=((shape[1]+sizeof(int))*shape[0]);
2009  field_types[i]=item_type0;
2010  opened_types_.insert(item_type1);
2011  opened_types_.insert(field_types[i]);
2012 
2013  }
2014 
2015  }else if(valtype==typeid(std::map<std::pair<int, std::string>, double>)){
2016  shape=shapes[i];
2017  if(shape[0]<1&&shape[3]>=1){
2018  dbtypes[i]=VL_MAP_PAIR_INT_STRING_DOUBLE;
2019  hid_t item_type0;
2020  hid_t item_type1key;
2021  hid_t item_type2first;
2022  item_type2first=H5T_NATIVE_INT;
2023  hid_t item_type2second;
2024  item_type2second=sha1_type_;
2025  hid_t item_type2;
2026  item_type2=H5Tcreate(H5T_COMPOUND,sizeof(int)+CYCLUS_SHA1_SIZE);
2027  H5Tinsert(item_type2,"first",0,item_type2first);
2028  H5Tinsert(item_type2,"second",0+sizeof(int),item_type2second);
2029  hid_t item_type1val;
2030  item_type1val=H5T_NATIVE_DOUBLE;
2031  hid_t item_type1;
2032  item_type1=H5Tcreate(H5T_COMPOUND,((sizeof(int)+CYCLUS_SHA1_SIZE))+sizeof(double));
2033  H5Tinsert(item_type1,"key",0,item_type2);
2034  H5Tinsert(item_type1,"val",0+((sizeof(int)+CYCLUS_SHA1_SIZE)),item_type1val);
2035  dst_sizes[i]=(CYCLUS_SHA1_SIZE);
2036  field_types[i]=sha1_type_;
2037  if(vldts_.count(VL_MAP_PAIR_INT_STRING_DOUBLE)==0){
2038  vldts_[VL_MAP_PAIR_INT_STRING_DOUBLE]=H5Tvlen_create(item_type1);
2039  opened_types_.insert(vldts_[VL_MAP_PAIR_INT_STRING_DOUBLE]);
2040 
2041  }
2042 
2043  }else if(shape[0]>=1&&shape[3]<1){
2044  dbtypes[i]=MAP_PAIR_INT_VL_STRING_DOUBLE;
2045  hid_t item_type0;
2046  hsize_t shape01=shape[0];
2047  hid_t item_type1key;
2048  hid_t item_type2first;
2049  item_type2first=H5T_NATIVE_INT;
2050  hid_t item_type2second;
2051  item_type2second=sha1_type_;
2052  hid_t item_type2;
2053  item_type2=H5Tcreate(H5T_COMPOUND,sizeof(int)+CYCLUS_SHA1_SIZE);
2054  H5Tinsert(item_type2,"first",0,item_type2first);
2055  H5Tinsert(item_type2,"second",0+sizeof(int),item_type2second);
2056  hid_t item_type1val;
2057  item_type1val=H5T_NATIVE_DOUBLE;
2058  hid_t item_type1;
2059  item_type1=H5Tcreate(H5T_COMPOUND,((sizeof(int)+CYCLUS_SHA1_SIZE))+sizeof(double));
2060  H5Tinsert(item_type1,"key",0,item_type2);
2061  H5Tinsert(item_type1,"val",0+((sizeof(int)+CYCLUS_SHA1_SIZE)),item_type1val);
2062  item_type0=H5Tarray_create2(item_type1,1,&shape01);
2063  dst_sizes[i]=((((sizeof(int)+CYCLUS_SHA1_SIZE))+sizeof(double))*shape[0]);
2064  field_types[i]=item_type0;
2065  opened_types_.insert(item_type2);
2066  opened_types_.insert(item_type1);
2067  opened_types_.insert(field_types[i]);
2068 
2069  }else if(shape.empty()||shape[0]<1&&shape[3]<1){
2071  hid_t item_type0;
2072  hid_t item_type1key;
2073  hid_t item_type2first;
2074  item_type2first=H5T_NATIVE_INT;
2075  hid_t item_type2second;
2076  item_type2second=sha1_type_;
2077  hid_t item_type2;
2078  item_type2=H5Tcreate(H5T_COMPOUND,sizeof(int)+CYCLUS_SHA1_SIZE);
2079  H5Tinsert(item_type2,"first",0,item_type2first);
2080  H5Tinsert(item_type2,"second",0+sizeof(int),item_type2second);
2081  hid_t item_type1val;
2082  item_type1val=H5T_NATIVE_DOUBLE;
2083  hid_t item_type1;
2084  item_type1=H5Tcreate(H5T_COMPOUND,((sizeof(int)+CYCLUS_SHA1_SIZE))+sizeof(double));
2085  H5Tinsert(item_type1,"key",0,item_type2);
2086  H5Tinsert(item_type1,"val",0+((sizeof(int)+CYCLUS_SHA1_SIZE)),item_type1val);
2087  dst_sizes[i]=(CYCLUS_SHA1_SIZE);
2088  field_types[i]=sha1_type_;
2089  if(vldts_.count(VL_MAP_PAIR_INT_VL_STRING_DOUBLE)==0){
2090  vldts_[VL_MAP_PAIR_INT_VL_STRING_DOUBLE]=H5Tvlen_create(item_type1);
2091  opened_types_.insert(vldts_[VL_MAP_PAIR_INT_VL_STRING_DOUBLE]);
2092 
2093  }
2094 
2095  }else{
2096  dbtypes[i]=MAP_PAIR_INT_STRING_DOUBLE;
2097  hid_t item_type0;
2098  hsize_t shape01=shape[0];
2099  hid_t item_type1key;
2100  hid_t item_type2first;
2101  item_type2first=H5T_NATIVE_INT;
2102  hid_t item_type2second;
2103  item_type2second=CreateFLStrType(shape[3]);
2104  hid_t item_type2;
2105  item_type2=H5Tcreate(H5T_COMPOUND,sizeof(int)+shape[3]);
2106  H5Tinsert(item_type2,"first",0,item_type2first);
2107  H5Tinsert(item_type2,"second",0+sizeof(int),item_type2second);
2108  hid_t item_type1val;
2109  item_type1val=H5T_NATIVE_DOUBLE;
2110  hid_t item_type1;
2111  item_type1=H5Tcreate(H5T_COMPOUND,((sizeof(int)+shape[3]))+sizeof(double));
2112  H5Tinsert(item_type1,"key",0,item_type2);
2113  H5Tinsert(item_type1,"val",0+((sizeof(int)+shape[3])),item_type1val);
2114  item_type0=H5Tarray_create2(item_type1,1,&shape01);
2115  dst_sizes[i]=((((sizeof(int)+shape[3]))+sizeof(double))*shape[0]);
2116  field_types[i]=item_type0;
2117  opened_types_.insert(item_type2);
2118  opened_types_.insert(item_type1);
2119  opened_types_.insert(field_types[i]);
2120 
2121  }
2122 
2123  }else if(valtype==typeid(std::pair<int, int>)){
2124  shape=shapes[i];
2125  dbtypes[i]=PAIR_INT_INT;
2126  hid_t item_type0;
2127  hid_t item_type1first;
2128  item_type1first=H5T_NATIVE_INT;
2129  hid_t item_type1second;
2130  item_type1second=H5T_NATIVE_INT;
2131  hid_t item_type1;
2132  item_type1=H5Tcreate(H5T_COMPOUND,sizeof(int)+sizeof(int));
2133  H5Tinsert(item_type1,"first",0,item_type1first);
2134  H5Tinsert(item_type1,"second",0+sizeof(int),item_type1second);
2135  dst_sizes[i]=((sizeof(int)+sizeof(int)));
2136  field_types[i]=item_type1;
2137  opened_types_.insert(field_types[i]);
2138 
2139  }else if(valtype==typeid(std::vector<double>)){
2140  shape=shapes[i];
2141  if(shape.empty()||shape[0]<1){
2142  dbtypes[i]=VL_VECTOR_DOUBLE;
2143  hid_t item_type0;
2144  hid_t item_type1elem;
2145  item_type1elem=H5T_NATIVE_DOUBLE;
2146  dst_sizes[i]=(CYCLUS_SHA1_SIZE);
2147  field_types[i]=sha1_type_;
2148  if(vldts_.count(VL_VECTOR_DOUBLE)==0){
2149  vldts_[VL_VECTOR_DOUBLE]=H5Tvlen_create(item_type1elem);
2150  opened_types_.insert(vldts_[VL_VECTOR_DOUBLE]);
2151 
2152  }
2153 
2154  }else{
2155  dbtypes[i]=VECTOR_DOUBLE;
2156  hid_t item_type0;
2157  hsize_t shape01=shape[0];
2158  hid_t item_type1elem;
2159  item_type1elem=H5T_NATIVE_DOUBLE;
2160  item_type0=H5Tarray_create2(item_type1elem,1,&shape01);
2161  dst_sizes[i]=((sizeof(double))*shape[0]);
2162  field_types[i]=item_type0;
2163  opened_types_.insert(field_types[i]);
2164 
2165  }
2166 
2167  }else if(valtype==typeid(boost::uuids::uuid)){
2168  shape=shapes[i];
2169  dbtypes[i]=UUID;
2170  hid_t item_type0;
2171  item_type0=uuid_type_;
2172  dst_sizes[i]=CYCLUS_UUID_SIZE;
2173  field_types[i]=item_type0;
2174 
2175  }else if(valtype==typeid(int)){
2176  shape=shapes[i];
2177  dbtypes[i]=INT;
2178  hid_t item_type0;
2179  item_type0=H5T_NATIVE_INT;
2180  dst_sizes[i]=sizeof(int);
2181  field_types[i]=item_type0;
2182 
2183  }else if(valtype==typeid(double)){
2184  shape=shapes[i];
2185  dbtypes[i]=DOUBLE;
2186  hid_t item_type0;
2187  item_type0=H5T_NATIVE_DOUBLE;
2188  dst_sizes[i]=sizeof(double);
2189  field_types[i]=item_type0;
2190 
2191  }else if(valtype==typeid(float)){
2192  shape=shapes[i];
2193  dbtypes[i]=FLOAT;
2194  hid_t item_type0;
2195  item_type0=H5T_NATIVE_FLOAT;
2196  dst_sizes[i]=sizeof(float);
2197  field_types[i]=item_type0;
2198 
2199  }else if(valtype==typeid(std::map<int, double>)){
2200  shape=shapes[i];
2201  if(shape.empty()||shape[0]<1){
2202  dbtypes[i]=VL_MAP_INT_DOUBLE;
2203  hid_t item_type0;
2204  hid_t item_type1key;
2205  item_type1key=H5T_NATIVE_INT;
2206  hid_t item_type1val;
2207  item_type1val=H5T_NATIVE_DOUBLE;
2208  hid_t item_type1;
2209  item_type1=H5Tcreate(H5T_COMPOUND,sizeof(int)+sizeof(double));
2210  H5Tinsert(item_type1,"key",0,item_type1key);
2211  H5Tinsert(item_type1,"val",0+sizeof(int),item_type1val);
2212  dst_sizes[i]=(CYCLUS_SHA1_SIZE);
2213  field_types[i]=sha1_type_;
2214  if(vldts_.count(VL_MAP_INT_DOUBLE)==0){
2215  vldts_[VL_MAP_INT_DOUBLE]=H5Tvlen_create(item_type1);
2216  opened_types_.insert(vldts_[VL_MAP_INT_DOUBLE]);
2217 
2218  }
2219 
2220  }else{
2221  dbtypes[i]=MAP_INT_DOUBLE;
2222  hid_t item_type0;
2223  hsize_t shape01=shape[0];
2224  hid_t item_type1key;
2225  item_type1key=H5T_NATIVE_INT;
2226  hid_t item_type1val;
2227  item_type1val=H5T_NATIVE_DOUBLE;
2228  hid_t item_type1;
2229  item_type1=H5Tcreate(H5T_COMPOUND,sizeof(int)+sizeof(double));
2230  H5Tinsert(item_type1,"key",0,item_type1key);
2231  H5Tinsert(item_type1,"val",0+sizeof(int),item_type1val);
2232  item_type0=H5Tarray_create2(item_type1,1,&shape01);
2233  dst_sizes[i]=((sizeof(int)+sizeof(double))*shape[0]);
2234  field_types[i]=item_type0;
2235  opened_types_.insert(item_type1);
2236  opened_types_.insert(field_types[i]);
2237 
2238  }
2239 
2240  }else if(valtype==typeid(std::set<int>)){
2241  shape=shapes[i];
2242  if(shape.empty()||shape[0]<1){
2243  dbtypes[i]=VL_SET_INT;
2244  hid_t item_type0;
2245  hid_t item_type1elem;
2246  item_type1elem=H5T_NATIVE_INT;
2247  dst_sizes[i]=(CYCLUS_SHA1_SIZE);
2248  field_types[i]=sha1_type_;
2249  if(vldts_.count(VL_SET_INT)==0){
2250  vldts_[VL_SET_INT]=H5Tvlen_create(item_type1elem);
2251  opened_types_.insert(vldts_[VL_SET_INT]);
2252 
2253  }
2254 
2255  }else{
2256  dbtypes[i]=SET_INT;
2257  hid_t item_type0;
2258  hsize_t shape01=shape[0];
2259  hid_t item_type1elem;
2260  item_type1elem=H5T_NATIVE_INT;
2261  item_type0=H5Tarray_create2(item_type1elem,1,&shape01);
2262  dst_sizes[i]=((sizeof(int))*shape[0]);
2263  field_types[i]=item_type0;
2264  opened_types_.insert(field_types[i]);
2265 
2266  }
2267 
2268  }else if(valtype==typeid(std::map<int, int>)){
2269  shape=shapes[i];
2270  if(shape.empty()||shape[0]<1){
2271  dbtypes[i]=VL_MAP_INT_INT;
2272  hid_t item_type0;
2273  hid_t item_type1key;
2274  item_type1key=H5T_NATIVE_INT;
2275  hid_t item_type1val;
2276  item_type1val=H5T_NATIVE_INT;
2277  hid_t item_type1;
2278  item_type1=H5Tcreate(H5T_COMPOUND,sizeof(int)+sizeof(int));
2279  H5Tinsert(item_type1,"key",0,item_type1key);
2280  H5Tinsert(item_type1,"val",0+sizeof(int),item_type1val);
2281  dst_sizes[i]=(CYCLUS_SHA1_SIZE);
2282  field_types[i]=sha1_type_;
2283  if(vldts_.count(VL_MAP_INT_INT)==0){
2284  vldts_[VL_MAP_INT_INT]=H5Tvlen_create(item_type1);
2285  opened_types_.insert(vldts_[VL_MAP_INT_INT]);
2286 
2287  }
2288 
2289  }else{
2290  dbtypes[i]=MAP_INT_INT;
2291  hid_t item_type0;
2292  hsize_t shape01=shape[0];
2293  hid_t item_type1key;
2294  item_type1key=H5T_NATIVE_INT;
2295  hid_t item_type1val;
2296  item_type1val=H5T_NATIVE_INT;
2297  hid_t item_type1;
2298  item_type1=H5Tcreate(H5T_COMPOUND,sizeof(int)+sizeof(int));
2299  H5Tinsert(item_type1,"key",0,item_type1key);
2300  H5Tinsert(item_type1,"val",0+sizeof(int),item_type1val);
2301  item_type0=H5Tarray_create2(item_type1,1,&shape01);
2302  dst_sizes[i]=((sizeof(int)+sizeof(int))*shape[0]);
2303  field_types[i]=item_type0;
2304  opened_types_.insert(item_type1);
2305  opened_types_.insert(field_types[i]);
2306 
2307  }
2308 
2309  }else if(valtype==typeid(std::map<int, std::string>)){
2310  shape=shapes[i];
2311  if(shape[0]<1&&shape[2]>=1){
2312  dbtypes[i]=VL_MAP_INT_STRING;
2313  hid_t item_type0;
2314  hid_t item_type1key;
2315  item_type1key=H5T_NATIVE_INT;
2316  hid_t item_type1val;
2317  item_type1val=sha1_type_;
2318  hid_t item_type1;
2319  item_type1=H5Tcreate(H5T_COMPOUND,sizeof(int)+CYCLUS_SHA1_SIZE);
2320  H5Tinsert(item_type1,"key",0,item_type1key);
2321  H5Tinsert(item_type1,"val",0+sizeof(int),item_type1val);
2322  dst_sizes[i]=(CYCLUS_SHA1_SIZE);
2323  field_types[i]=sha1_type_;
2324  if(vldts_.count(VL_MAP_INT_STRING)==0){
2325  vldts_[VL_MAP_INT_STRING]=H5Tvlen_create(item_type1);
2326  opened_types_.insert(vldts_[VL_MAP_INT_STRING]);
2327 
2328  }
2329 
2330  }else if(shape[0]>=1&&shape[2]<1){
2331  dbtypes[i]=MAP_INT_VL_STRING;
2332  hid_t item_type0;
2333  hsize_t shape01=shape[0];
2334  hid_t item_type1key;
2335  item_type1key=H5T_NATIVE_INT;
2336  hid_t item_type1val;
2337  item_type1val=sha1_type_;
2338  hid_t item_type1;
2339  item_type1=H5Tcreate(H5T_COMPOUND,sizeof(int)+CYCLUS_SHA1_SIZE);
2340  H5Tinsert(item_type1,"key",0,item_type1key);
2341  H5Tinsert(item_type1,"val",0+sizeof(int),item_type1val);
2342  item_type0=H5Tarray_create2(item_type1,1,&shape01);
2343  dst_sizes[i]=((sizeof(int)+CYCLUS_SHA1_SIZE)*shape[0]);
2344  field_types[i]=item_type0;
2345  opened_types_.insert(item_type1);
2346  opened_types_.insert(field_types[i]);
2347 
2348  }else if(shape.empty()||shape[0]<1&&shape[2]<1){
2349  dbtypes[i]=VL_MAP_INT_VL_STRING;
2350  hid_t item_type0;
2351  hid_t item_type1key;
2352  item_type1key=H5T_NATIVE_INT;
2353  hid_t item_type1val;
2354  item_type1val=sha1_type_;
2355  hid_t item_type1;
2356  item_type1=H5Tcreate(H5T_COMPOUND,sizeof(int)+CYCLUS_SHA1_SIZE);
2357  H5Tinsert(item_type1,"key",0,item_type1key);
2358  H5Tinsert(item_type1,"val",0+sizeof(int),item_type1val);
2359  dst_sizes[i]=(CYCLUS_SHA1_SIZE);
2360  field_types[i]=sha1_type_;
2361  if(vldts_.count(VL_MAP_INT_VL_STRING)==0){
2362  vldts_[VL_MAP_INT_VL_STRING]=H5Tvlen_create(item_type1);
2363  opened_types_.insert(vldts_[VL_MAP_INT_VL_STRING]);
2364 
2365  }
2366 
2367  }else{
2368  dbtypes[i]=MAP_INT_STRING;
2369  hid_t item_type0;
2370  hsize_t shape01=shape[0];
2371  hid_t item_type1key;
2372  item_type1key=H5T_NATIVE_INT;
2373  hid_t item_type1val;
2374  item_type1val=CreateFLStrType(shape[2]);
2375  hid_t item_type1;
2376  item_type1=H5Tcreate(H5T_COMPOUND,sizeof(int)+shape[2]);
2377  H5Tinsert(item_type1,"key",0,item_type1key);
2378  H5Tinsert(item_type1,"val",0+sizeof(int),item_type1val);
2379  item_type0=H5Tarray_create2(item_type1,1,&shape01);
2380  dst_sizes[i]=((sizeof(int)+shape[2])*shape[0]);
2381  field_types[i]=item_type0;
2382  opened_types_.insert(item_type1);
2383  opened_types_.insert(field_types[i]);
2384 
2385  }
2386 
2387  }else if(valtype==typeid(bool)){
2388  shape=shapes[i];
2389  dbtypes[i]=BOOL;
2390  hid_t item_type0;
2391  item_type0=H5T_NATIVE_CHAR;
2392  dst_sizes[i]=sizeof(char);
2393  field_types[i]=item_type0;
2394 
2395  }else if(valtype==typeid(cyclus::Blob)){
2396  shape=shapes[i];
2397  dbtypes[i]=BLOB;
2398  field_types[i]=sha1_type_;
2399  dst_sizes[i]=CYCLUS_SHA1_SIZE;
2400 
2401  }else if(valtype==typeid(std::map<std::string, std::string>)){
2402  shape=shapes[i];
2403  if(shape.empty()||shape[0]<1&&shape[1]<1&&shape[2]<1){
2404  dbtypes[i]=VL_MAP_VL_STRING_VL_STRING;
2405  hid_t item_type0;
2406  hid_t item_type1key;
2407  item_type1key=sha1_type_;
2408  hid_t item_type1val;
2409  item_type1val=sha1_type_;
2410  hid_t item_type1;
2411  item_type1=H5Tcreate(H5T_COMPOUND,CYCLUS_SHA1_SIZE+CYCLUS_SHA1_SIZE);
2412  H5Tinsert(item_type1,"key",0,item_type1key);
2413  H5Tinsert(item_type1,"val",0+CYCLUS_SHA1_SIZE,item_type1val);
2414  dst_sizes[i]=(CYCLUS_SHA1_SIZE);
2415  field_types[i]=sha1_type_;
2416  if(vldts_.count(VL_MAP_VL_STRING_VL_STRING)==0){
2417  vldts_[VL_MAP_VL_STRING_VL_STRING]=H5Tvlen_create(item_type1);
2418  opened_types_.insert(vldts_[VL_MAP_VL_STRING_VL_STRING]);
2419 
2420  }
2421 
2422  }else if(shape[0]>=1&&shape[1]<1&&shape[2]>=1){
2423  dbtypes[i]=MAP_VL_STRING_STRING;
2424  hid_t item_type0;
2425  hsize_t shape01=shape[0];
2426  hid_t item_type1key;
2427  item_type1key=sha1_type_;
2428  hid_t item_type1val;
2429  item_type1val=CreateFLStrType(shape[2]);
2430  hid_t item_type1;
2431  item_type1=H5Tcreate(H5T_COMPOUND,CYCLUS_SHA1_SIZE+shape[2]);
2432  H5Tinsert(item_type1,"key",0,item_type1key);
2433  H5Tinsert(item_type1,"val",0+CYCLUS_SHA1_SIZE,item_type1val);
2434  item_type0=H5Tarray_create2(item_type1,1,&shape01);
2435  dst_sizes[i]=((CYCLUS_SHA1_SIZE+shape[2])*shape[0]);
2436  field_types[i]=item_type0;
2437  opened_types_.insert(item_type1);
2438  opened_types_.insert(field_types[i]);
2439 
2440  }else if(shape[0]<1&&shape[1]>=1&&shape[2]<1){
2441  dbtypes[i]=VL_MAP_STRING_VL_STRING;
2442  hid_t item_type0;
2443  hid_t item_type1key;
2444  item_type1key=sha1_type_;
2445  hid_t item_type1val;
2446  item_type1val=sha1_type_;
2447  hid_t item_type1;
2448  item_type1=H5Tcreate(H5T_COMPOUND,CYCLUS_SHA1_SIZE+CYCLUS_SHA1_SIZE);
2449  H5Tinsert(item_type1,"key",0,item_type1key);
2450  H5Tinsert(item_type1,"val",0+CYCLUS_SHA1_SIZE,item_type1val);
2451  dst_sizes[i]=(CYCLUS_SHA1_SIZE);
2452  field_types[i]=sha1_type_;
2453  if(vldts_.count(VL_MAP_STRING_VL_STRING)==0){
2454  vldts_[VL_MAP_STRING_VL_STRING]=H5Tvlen_create(item_type1);
2455  opened_types_.insert(vldts_[VL_MAP_STRING_VL_STRING]);
2456 
2457  }
2458 
2459  }else if(shape[0]>=1&&shape[1]>=1&&shape[2]<1){
2460  dbtypes[i]=MAP_STRING_VL_STRING;
2461  hid_t item_type0;
2462  hsize_t shape01=shape[0];
2463  hid_t item_type1key;
2464  item_type1key=CreateFLStrType(shape[1]);
2465  hid_t item_type1val;
2466  item_type1val=sha1_type_;
2467  hid_t item_type1;
2468  item_type1=H5Tcreate(H5T_COMPOUND,shape[1]+CYCLUS_SHA1_SIZE);
2469  H5Tinsert(item_type1,"key",0,item_type1key);
2470  H5Tinsert(item_type1,"val",0+shape[1],item_type1val);
2471  item_type0=H5Tarray_create2(item_type1,1,&shape01);
2472  dst_sizes[i]=((shape[1]+CYCLUS_SHA1_SIZE)*shape[0]);
2473  field_types[i]=item_type0;
2474  opened_types_.insert(item_type1);
2475  opened_types_.insert(field_types[i]);
2476 
2477  }else if(shape[0]<1&&shape[1]<1&&shape[2]>=1){
2478  dbtypes[i]=VL_MAP_VL_STRING_STRING;
2479  hid_t item_type0;
2480  hid_t item_type1key;
2481  item_type1key=sha1_type_;
2482  hid_t item_type1val;
2483  item_type1val=sha1_type_;
2484  hid_t item_type1;
2485  item_type1=H5Tcreate(H5T_COMPOUND,CYCLUS_SHA1_SIZE+CYCLUS_SHA1_SIZE);
2486  H5Tinsert(item_type1,"key",0,item_type1key);
2487  H5Tinsert(item_type1,"val",0+CYCLUS_SHA1_SIZE,item_type1val);
2488  dst_sizes[i]=(CYCLUS_SHA1_SIZE);
2489  field_types[i]=sha1_type_;
2490  if(vldts_.count(VL_MAP_VL_STRING_STRING)==0){
2491  vldts_[VL_MAP_VL_STRING_STRING]=H5Tvlen_create(item_type1);
2492  opened_types_.insert(vldts_[VL_MAP_VL_STRING_STRING]);
2493 
2494  }
2495 
2496  }else if(shape[0]<1&&shape[1]>=1&&shape[2]>=1){
2497  dbtypes[i]=VL_MAP_STRING_STRING;
2498  hid_t item_type0;
2499  hid_t item_type1key;
2500  item_type1key=sha1_type_;
2501  hid_t item_type1val;
2502  item_type1val=sha1_type_;
2503  hid_t item_type1;
2504  item_type1=H5Tcreate(H5T_COMPOUND,CYCLUS_SHA1_SIZE+CYCLUS_SHA1_SIZE);
2505  H5Tinsert(item_type1,"key",0,item_type1key);
2506  H5Tinsert(item_type1,"val",0+CYCLUS_SHA1_SIZE,item_type1val);
2507  dst_sizes[i]=(CYCLUS_SHA1_SIZE);
2508  field_types[i]=sha1_type_;
2509  if(vldts_.count(VL_MAP_STRING_STRING)==0){
2510  vldts_[VL_MAP_STRING_STRING]=H5Tvlen_create(item_type1);
2511  opened_types_.insert(vldts_[VL_MAP_STRING_STRING]);
2512 
2513  }
2514 
2515  }else if(shape[0]>=1&&shape[1]<1&&shape[2]<1){
2516  dbtypes[i]=MAP_VL_STRING_VL_STRING;
2517  hid_t item_type0;
2518  hsize_t shape01=shape[0];
2519  hid_t item_type1key;
2520  item_type1key=sha1_type_;
2521  hid_t item_type1val;
2522  item_type1val=sha1_type_;
2523  hid_t item_type1;
2524  item_type1=H5Tcreate(H5T_COMPOUND,CYCLUS_SHA1_SIZE+CYCLUS_SHA1_SIZE);
2525  H5Tinsert(item_type1,"key",0,item_type1key);
2526  H5Tinsert(item_type1,"val",0+CYCLUS_SHA1_SIZE,item_type1val);
2527  item_type0=H5Tarray_create2(item_type1,1,&shape01);
2528  dst_sizes[i]=((CYCLUS_SHA1_SIZE+CYCLUS_SHA1_SIZE)*shape[0]);
2529  field_types[i]=item_type0;
2530  opened_types_.insert(item_type1);
2531  opened_types_.insert(field_types[i]);
2532 
2533  }else{
2534  dbtypes[i]=MAP_STRING_STRING;
2535  hid_t item_type0;
2536  hsize_t shape01=shape[0];
2537  hid_t item_type1key;
2538  item_type1key=CreateFLStrType(shape[1]);
2539  hid_t item_type1val;
2540  item_type1val=CreateFLStrType(shape[2]);
2541  hid_t item_type1;
2542  item_type1=H5Tcreate(H5T_COMPOUND,shape[1]+shape[2]);
2543  H5Tinsert(item_type1,"key",0,item_type1key);
2544  H5Tinsert(item_type1,"val",0+shape[1],item_type1val);
2545  item_type0=H5Tarray_create2(item_type1,1,&shape01);
2546  dst_sizes[i]=((shape[1]+shape[2])*shape[0]);
2547  field_types[i]=item_type0;
2548  opened_types_.insert(item_type1);
2549  opened_types_.insert(field_types[i]);
2550 
2551  }
2552 
2553  }else if(valtype==typeid(std::list<int>)){
2554  shape=shapes[i];
2555  if(shape.empty()||shape[0]<1){
2556  dbtypes[i]=VL_LIST_INT;
2557  hid_t item_type0;
2558  hid_t item_type1elem;
2559  item_type1elem=H5T_NATIVE_INT;
2560  dst_sizes[i]=(CYCLUS_SHA1_SIZE);
2561  field_types[i]=sha1_type_;
2562  if(vldts_.count(VL_LIST_INT)==0){
2563  vldts_[VL_LIST_INT]=H5Tvlen_create(item_type1elem);
2564  opened_types_.insert(vldts_[VL_LIST_INT]);
2565 
2566  }
2567 
2568  }else{
2569  dbtypes[i]=LIST_INT;
2570  hid_t item_type0;
2571  hsize_t shape01=shape[0];
2572  hid_t item_type1elem;
2573  item_type1elem=H5T_NATIVE_INT;
2574  item_type0=H5Tarray_create2(item_type1elem,1,&shape01);
2575  dst_sizes[i]=((sizeof(int))*shape[0]);
2576  field_types[i]=item_type0;
2577  opened_types_.insert(field_types[i]);
2578 
2579  }
2580 
2581  }else{
2582  throw IOError("the type for column '"+std::string(field_names[i])+"' is not yet supported in HDF5.");
2583  }
2584 
2585 
2586  dst_size += dst_sizes[i];
2587  }
2588 
2589  std::string titlestr = d->title();
2590  const char* title = titlestr.c_str();
2591  int compress = 1;
2592  int chunk_size = 1024;
2593  void* fill_data = NULL;
2594  void* data = NULL;
2595 
2596  // Make the table
2597  status = H5TBmake_table(title, file_, title, nvals, 0, dst_size,
2598  field_names, dst_offset, field_types, chunk_size,
2599  fill_data, compress, data);
2600  if (status < 0) {
2601  std::stringstream ss;
2602  ss << "Failed to create HDF5 table:\n" \
2603  << " file " << path_ << "\n" \
2604  << " table " << title << "\n" \
2605  << " chunksize " << chunk_size << "\n" \
2606  << " rowsize " << dst_size << "\n";
2607  for (int i = 0; i < nvals; ++i) {
2608  ss << " #" << i << " " << field_names[i] << "\n" \
2609  << " dbtype: " << dbtypes[i] << "\n" \
2610  << " h5type: " << field_types[i] << "\n" \
2611  << " size: " << dst_sizes[i] << "\n" \
2612  << " offset: " << dst_offset[i] << "\n";
2613  }
2614  throw IOError(ss.str());
2615  }
2616 
2617  // add dbtypes attribute
2618  hid_t tb_set = H5Dopen2(file_, title, H5P_DEFAULT);
2619  hid_t attr_space = H5Screate_simple(1, &nvals, &nvals);
2620  hid_t dbtypes_attr = H5Acreate2(tb_set, "cyclus_dbtypes", H5T_NATIVE_INT,
2621  attr_space, H5P_DEFAULT, H5P_DEFAULT);
2622  H5Awrite(dbtypes_attr, H5T_NATIVE_INT, dbtypes);
2623  H5Aclose(dbtypes_attr);
2624  H5Sclose(attr_space);
2625  H5Dclose(tb_set);
2626 
2627  // record everything for later
2628  col_offsets_[d->title()] = dst_offset;
2629  schema_sizes_[d->title()] = dst_size;
2630  col_sizes_[d->title()] = dst_sizes;
2631  schemas_[d->title()] = dbtypes;
2632 }
2633 
2634 std::map<std::string, DbTypes> Hdf5Back::ColumnTypes(std::string table) {
2635  using std::string;
2636  int i;
2637  char* colname;
2638  hid_t dset = H5Dopen2(file_, table.c_str(), H5P_DEFAULT);
2639  hid_t dt = H5Dget_type(dset);
2640  hsize_t ncols = H5Tget_nmembers(dt);
2641  string fieldname;
2642  string fieldtype;
2643  LoadTableTypes(table, dset, ncols);
2644  DbTypes* dbtypes = schemas_[table];
2645 
2646  // create return value
2647  std::map<string, DbTypes> rtn;
2648  for (i = 0; i < ncols; ++i) {
2649  colname = H5Tget_member_name(dt, i);
2650  fieldname = string(colname);
2651  free(colname);
2652  rtn[fieldname] = dbtypes[i];
2653  }
2654 
2655  // close and return
2656  H5Tclose(dt);
2657  H5Dclose(dset);
2658  return rtn;
2659 }
2660 
2661 std::set<std::string> Hdf5Back::Tables() {
2662  using std::set;
2663  using std::string;
2664  set<string> rtn;
2665  hsize_t i;
2666  hsize_t n;
2667  ssize_t namelen;
2668  char name[500];
2669  H5G_info_t root_info;
2670  hid_t root = H5Gopen(file_, "/", H5P_DEFAULT);
2671  herr_t err = H5Gget_info(root, &root_info);
2672  for (i = 0; i < root_info.nlinks; ++i) {
2673  namelen = H5Lget_name_by_idx(root, ".", H5_INDEX_NAME, H5_ITER_NATIVE, i,
2674  NULL, 0, H5P_DEFAULT);
2675  H5Lget_name_by_idx(root, ".", H5_INDEX_NAME, H5_ITER_NATIVE, i,
2676  name, namelen+1, H5P_DEFAULT);
2677  rtn.insert(string(name, namelen));
2678  }
2679  H5Gclose(root);
2680  return rtn;
2681 }
2682 
2683 void Hdf5Back::WriteGroup(DatumList& group) {
2684  std::string title = group.front()->title();
2685  const char * c_title = title.c_str();
2686 
2687  size_t* offsets = col_offsets_[title];
2688  size_t* sizes = col_sizes_[title];
2689  size_t rowsize = schema_sizes_[title];
2690 
2691  char* buf = new char[group.size() * rowsize];
2692  FillBuf(title, buf, group, sizes, rowsize);
2693 
2694  // We cannot do the simple thing (append_records) here because of a bug in
2695  // H5TB where it stupidly tries to reconstruct the datatype in memory from
2696  // what it read in on disk. This works in most cases but failed where the table
2697  // had a column which is an array of a compound datatype of non-homogenous
2698  // fields (eg MAP_INT_DOUBLE). The fix here just uses the datatype present on
2699  // disk - which is what we wanted anyway!
2700  //herr_t status = H5TBappend_records(file_, title.c_str(), group.size(), rowsize,
2701  // offsets, sizes, buf);
2702  herr_t status;
2703  hid_t dset = H5Dopen2(file_, title.c_str(), H5P_DEFAULT);
2704  hid_t dtype = H5Dget_type(dset);
2705  hsize_t nrecords_add = group.size();
2706  hsize_t nrecords_orig;
2707  hsize_t nfields;
2708  hsize_t dims[1];
2709  hsize_t offset[1];
2710  hsize_t count[1];
2711  H5TBget_table_info(file_, c_title, &nfields, &nrecords_orig);
2712  dims[0] = nrecords_add + nrecords_orig;
2713  offset[0] = nrecords_orig;
2714  count[0] = nrecords_add;
2715 
2716  status = H5Dset_extent(dset, dims);
2717  hid_t dspace = H5Dget_space(dset);
2718  hid_t memspace = H5Screate_simple(1, count, NULL);
2719  status = H5Sselect_hyperslab(dspace, H5S_SELECT_SET, offset, NULL, count, NULL);
2720  status = H5Dwrite(dset, dtype, memspace, dspace, H5P_DEFAULT, buf);
2721 
2722  if (status < 0) {
2723  std::stringstream ss;
2724  ss << "Failed to write to the HDF5 table:\n" \
2725  << " file " << path_ << "\n" \
2726  << " table " << title << "\n" \
2727  << " num. rows " << group.size() << "\n"
2728  << " rowsize " << rowsize << "\n";
2729  for (int i = 0; i < group.front()->vals().size(); ++i) {
2730  ss << " # Column " << i << "\n" \
2731  << " dbtype: " << schemas_[title][i] << "\n" \
2732  << " size: " << sizes[i] << "\n" \
2733  << " offset: " << offsets[i] << "\n";
2734  }
2735  throw IOError(ss.str());
2736  }
2737 
2738  H5Sclose(memspace);
2739  H5Sclose(dspace);
2740  H5Tclose(dtype);
2741  H5Dclose(dset);
2742  delete[] buf;
2743 }
2744 
2745 template <typename T, DbTypes U>
2746 Digest Hdf5Back::VLWrite(const T& x) {
2747  hasher_.Clear();
2748  hasher_.Update(x);
2749  Digest key = hasher_.digest();
2750  hid_t keysds = VLDataset(U, true);
2751  hid_t valsds = VLDataset(U, false);
2752  if (vlkeys_[U].count(key) == 1)
2753  return key;
2754  hvl_t buf = VLValToBuf(x);
2755  AppendVLKey(keysds, U, key);
2756  InsertVLVal(valsds, U, key, buf);
2757  return key;
2758 }
2759 
2760 template <>
2761 Digest Hdf5Back::VLWrite<std::string, VL_STRING>(const std::string& x) {
2762  hasher_.Clear();
2763  hasher_.Update(x);
2764  Digest key = hasher_.digest();
2765  hid_t keysds = VLDataset(VL_STRING, true);
2766  hid_t valsds = VLDataset(VL_STRING, false);
2767  if (vlkeys_[VL_STRING].count(key) == 1)
2768  return key;
2769  AppendVLKey(keysds, VL_STRING, key);
2770  InsertVLVal(valsds, VL_STRING, key, x);
2771  return key;
2772 }
2773 
2774 template <>
2775 Digest Hdf5Back::VLWrite<Blob, BLOB>(const Blob& x) {
2776  hasher_.Clear();
2777  hasher_.Update(x);
2778  Digest key = hasher_.digest();
2779  hid_t keysds = VLDataset(BLOB, true);
2780  hid_t valsds = VLDataset(BLOB, false);
2781  if (vlkeys_[BLOB].count(key) == 1)
2782  return key;
2783  AppendVLKey(keysds, BLOB, key);
2784  InsertVLVal(valsds, BLOB, key, x.str());
2785  return key;
2786 }
2787 
2788 void Hdf5Back::FillBuf(std::string title, char* buf, DatumList& group,
2789  size_t* sizes, size_t rowsize) {
2790  using std::min;
2791  using std::string;
2792  using std::vector;
2793  using std::set;
2794  using std::list;
2795  using std::pair;
2796  using std::map;
2797  Datum::Vals vals;
2798  Datum::Shape shape;
2799  Datum::Shapes shapes;
2800  Datum::Vals header = group.front()->vals();
2801  int ncols = header.size();
2802  DbTypes* dbtypes = schemas_[title];
2803 
2804  size_t offset = 0;
2805  const void* val;
2806  size_t fieldlen;
2807  size_t valuelen;
2808  DatumList::iterator it;
2809  for (it = group.begin(); it != group.end(); ++it) {
2810  vals = (*it)->vals();
2811  shapes = (*it)->shapes();
2812  for (int col = 0; col < ncols; ++col) {
2813  const boost::spirit::hold_any* a = &(vals[col].second);
2814  switch (dbtypes[col]) {
2815  case BOOL:
2816  case INT:
2817  case FLOAT:
2818  case DOUBLE: {
2819  val = a->castsmallvoid();
2820  memcpy(buf + offset, val, sizes[col]);
2821  break;
2822  }
2823  case STRING: {
2824  const std::string s = a->cast<std::string>();
2825  fieldlen = sizes[col];
2826  valuelen = std::min(s.size(), fieldlen);
2827  memcpy(buf + offset, s.c_str(), valuelen);
2828  memset(buf + offset + valuelen, 0, fieldlen - valuelen);
2829  break;
2830  }
2831  case VL_STRING: {
2832  Digest key = VLWrite<std::string, VL_STRING>(a);
2833  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
2834  break;
2835  }
2836  case BLOB: {
2837  Digest key = VLWrite<Blob, BLOB>(a);
2838  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
2839  break;
2840  }
2841  case UUID: {
2842  boost::uuids::uuid uuid = a->cast<boost::uuids::uuid>();
2843  memcpy(buf + offset, &uuid, CYCLUS_UUID_SIZE);
2844  break;
2845  }
2846  case VECTOR_INT: {
2847  std::vector<int> val = a->cast<std::vector<int> >();
2848  fieldlen = sizes[col];
2849  valuelen = std::min(val.size() * sizeof(int), fieldlen);
2850  memcpy(buf + offset, &val[0], valuelen);
2851  memset(buf + offset + valuelen, 0, fieldlen - valuelen);
2852  break;
2853  }
2854  case VL_VECTOR_INT: {
2855  Digest key = VLWrite<std::vector<int>, VL_VECTOR_INT>(a);
2856  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
2857  break;
2858  }
2859  case VECTOR_FLOAT: {
2860  std::vector<float> val = a->cast<std::vector<float> >();
2861  fieldlen = sizes[col];
2862  valuelen = std::min(val.size() * sizeof(float), fieldlen);
2863  memcpy(buf + offset, &val[0], valuelen);
2864  memset(buf + offset + valuelen, 0, fieldlen - valuelen);
2865  break;
2866  }
2867  case VL_VECTOR_FLOAT: {
2868  Digest key = VLWrite<std::vector<float>, VL_VECTOR_FLOAT>(a);
2869  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
2870  break;
2871  }
2872  case VECTOR_DOUBLE: {
2873  std::vector<double> val = a->cast<std::vector<double> >();
2874  fieldlen = sizes[col];
2875  valuelen = std::min(val.size() * sizeof(double), fieldlen);
2876  memcpy(buf + offset, &val[0], valuelen);
2877  memset(buf + offset + valuelen, 0, fieldlen - valuelen);
2878  break;
2879  }
2880  case VL_VECTOR_DOUBLE: {
2881  Digest key = VLWrite<std::vector<double>, VL_VECTOR_DOUBLE>(a);
2882  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
2883  break;
2884  }
2885  case VECTOR_STRING: {
2886  vector<string> val = a->cast<vector<string> >();
2887  shape = shapes[col];
2888  fieldlen = shape[1];
2889  unsigned int cnt = 0;
2890  for (; cnt < val.size(); ++cnt) {
2891  valuelen = std::min(val[cnt].size(), fieldlen);
2892  memcpy(buf + offset + fieldlen*cnt, val[cnt].c_str(), valuelen);
2893  memset(buf + offset + fieldlen*cnt + valuelen, 0, fieldlen - valuelen);
2894  }
2895  memset(buf + offset + fieldlen*cnt, 0, fieldlen * (shape[0] - cnt));
2896  break;
2897  }
2898  case VECTOR_VL_STRING: {
2899  vector<string> val = a->cast<vector<string> >();
2900  Digest key;
2901  unsigned int cnt = 0;
2902  string s;
2903  for (; cnt < val.size(); ++cnt) {
2904  key = VLWrite<string, VL_STRING>(val[cnt]);
2905  memcpy(buf + offset + CYCLUS_SHA1_SIZE*cnt, key.val, CYCLUS_SHA1_SIZE);
2906  }
2907  memset(buf + offset + CYCLUS_SHA1_SIZE*cnt, 0, CYCLUS_SHA1_SIZE * (val.size() - cnt));
2908  break;
2909  }
2910  case VL_VECTOR_STRING: {
2911  shape = shapes[col];
2912  size_t strlen = shape[1];
2913  vector<string> givenval = a->cast<vector<string> >();
2914  vector<string> val = vector<string>(givenval.size());
2915  unsigned int cnt = 0;
2916  // ensure string is of specified length
2917  for (; cnt < givenval.size(); ++cnt)
2918  val[cnt] = string(givenval[cnt], 0, strlen);
2919  Digest key = VLWrite<vector<string>, VL_VECTOR_STRING>(val);
2920  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
2921  break;
2922  }
2923  case VL_VECTOR_VL_STRING: {
2924  Digest key = VLWrite<vector<string>, VL_VECTOR_VL_STRING>(a);
2925  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
2926  break;
2927  }
2928  case SET_INT: {
2929  std::set<int> val = a->cast<std::set<int> >();
2930  fieldlen = sizes[col];
2931  valuelen = std::min(val.size() * sizeof(int), fieldlen);
2932  unsigned int cnt = 0;
2933  for (std::set<int>::iterator sit = val.begin(); sit != val.end(); ++sit) {
2934  memcpy(buf + offset + cnt*sizeof(int), &(*sit), sizeof(int));
2935  ++cnt;
2936  }
2937  memset(buf + offset + valuelen, 0, fieldlen - valuelen);
2938  break;
2939  }
2940  case VL_SET_INT: {
2941  Digest key = VLWrite<std::set<int>, VL_SET_INT>(a);
2942  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
2943  break;
2944  }
2945  case SET_STRING: {
2946  set<string> val = a->cast<set<string> >();
2947  shape = shapes[col];
2948  fieldlen = shape[1];
2949  unsigned int cnt = 0;
2950  for (set<string>::iterator sit = val.begin(); sit != val.end(); ++sit) {
2951  valuelen = std::min(sit->size(), fieldlen);
2952  memcpy(buf + offset + fieldlen*cnt, sit->c_str(), valuelen);
2953  memset(buf + offset + fieldlen*cnt + valuelen, 0, fieldlen - valuelen);
2954  ++cnt;
2955  }
2956  memset(buf + offset + fieldlen*cnt, 0, fieldlen * (shape[0] - cnt));
2957  break;
2958  }
2959  case SET_VL_STRING: {
2960  set<string> val = a->cast<set<string> >();
2961  Digest key;
2962  unsigned int cnt = 0;
2963  for (set<string>::iterator sit = val.begin(); sit != val.end(); ++sit) {
2964  key = VLWrite<string, VL_STRING>(*sit);
2965  memcpy(buf + offset + CYCLUS_SHA1_SIZE*cnt, key.val, CYCLUS_SHA1_SIZE);
2966  ++cnt;
2967  }
2968  memset(buf+offset+CYCLUS_SHA1_SIZE*cnt, 0, CYCLUS_SHA1_SIZE*(val.size()-cnt));
2969  break;
2970  }
2971  case VL_SET_STRING: {
2972  shape = shapes[col];
2973  size_t strlen = shape[1];
2974  set<string> givenval = a->cast<set<string> >();
2975  set<string> val;
2976  // ensure string is of specified length
2977  for (set<string>::iterator sit = givenval.begin(); sit != givenval.end(); ++sit)
2978  val.insert(string((*sit), 0, strlen));
2979  Digest key = VLWrite<set<string>, VL_SET_STRING>(val);
2980  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
2981  break;
2982  }
2983  case VL_SET_VL_STRING: {
2984  Digest key = VLWrite<set<string>, VL_SET_VL_STRING>(a);
2985  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
2986  break;
2987  }
2988  case LIST_INT: {
2989  std::list<int> val = a->cast<std::list<int> >();
2990  fieldlen = sizes[col];
2991  valuelen = std::min(val.size() * sizeof(int), fieldlen);
2992  unsigned int cnt = 0;
2993  std::list<int>::iterator valit = val.begin();
2994  for (; valit != val.end(); ++valit) {
2995  memcpy(buf + offset + cnt*sizeof(int), &(*valit), sizeof(int));
2996  ++cnt;
2997  }
2998  memset(buf + offset + valuelen, 0, fieldlen - valuelen);
2999  break;
3000  }
3001  case VL_LIST_INT: {
3002  Digest key = VLWrite<std::list<int>, VL_LIST_INT>(a);
3003  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
3004  break;
3005  }
3006  case LIST_STRING: {
3007  list<string> val = a->cast<list<string> >();
3008  shape = shapes[col];
3009  fieldlen = shape[1];
3010  unsigned int cnt = 0;
3011  list<string>::iterator valit = val.begin();
3012  for (; valit != val.end(); ++valit) {
3013  valuelen = std::min(valit->size(), fieldlen);
3014  memcpy(buf + offset + fieldlen*cnt, valit->c_str(), valuelen);
3015  memset(buf + offset + fieldlen*cnt + valuelen, 0, fieldlen - valuelen);
3016  ++cnt;
3017  }
3018  memset(buf + offset + fieldlen*cnt, 0, fieldlen * (shape[0] - cnt));
3019  break;
3020  }
3021  case LIST_VL_STRING: {
3022  list<string> val = a->cast<list<string> >();
3023  Digest key;
3024  unsigned int cnt = 0;
3025  list<string>::iterator valit = val.begin();
3026  for (; valit != val.end(); ++valit) {
3027  key = VLWrite<string, VL_STRING>(*valit);
3028  memcpy(buf + offset + CYCLUS_SHA1_SIZE*cnt, key.val, CYCLUS_SHA1_SIZE);
3029  ++cnt;
3030  }
3031  memset(buf+offset+CYCLUS_SHA1_SIZE*cnt, 0, CYCLUS_SHA1_SIZE*(val.size()-cnt));
3032  break;
3033  }
3034  case VL_LIST_STRING: {
3035  shape = shapes[col];
3036  size_t strlen = shape[1];
3037  list<string> givenval = a->cast<list<string> >();
3038  list<string> val;
3039  // ensure string is of specified length
3040  list<string>::iterator valit = givenval.begin();
3041  for (; valit != givenval.end(); ++valit)
3042  val.push_back(string((*valit), 0, strlen));
3043  Digest key = VLWrite<list<string>, VL_LIST_STRING>(val);
3044  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
3045  break;
3046  }
3047  case VL_LIST_VL_STRING: {
3048  Digest key = VLWrite<list<string>, VL_LIST_VL_STRING>(a);
3049  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
3050  break;
3051  }
3052  case PAIR_INT_INT: {
3053  std::pair<int, int> val = a->cast<std::pair<int, int> >();
3054  memcpy(buf + offset, &(val.first), sizeof(int));
3055  memcpy(buf + offset + sizeof(int), &(val.second), sizeof(int));
3056  break;
3057  }
3058  case PAIR_INT_STRING: {
3059  pair<int, string> val = a->cast<pair<int, string> >();
3060  shape = shapes[col];
3061  int strlen = shape[2];
3062  fieldlen = sizeof(int) + strlen;
3063  memcpy(buf + offset, &(val.first), sizeof(int));
3064  valuelen = std::min(static_cast<int>(val.second.size()), strlen);
3065  memcpy(buf + offset + sizeof(int), val.second.c_str(), valuelen);
3066  memset(buf + offset + sizeof(int) + valuelen, 0, strlen - valuelen);
3067  break;
3068  }
3069  case PAIR_INT_VL_STRING: {
3070  pair<int, string> val = a->cast<pair<int, string> >();
3071  Digest valhash;
3072  fieldlen = sizeof(int) + CYCLUS_SHA1_SIZE;
3073  memcpy(buf + offset, &(val.first), sizeof(int));
3074  valhash = VLWrite<string, VL_STRING>(val.second);
3075  memcpy(buf + offset + sizeof(int), valhash.val, CYCLUS_SHA1_SIZE);
3076  break;
3077  }
3078  case MAP_INT_INT: {
3079  map<int, int> val = a->cast<map<int, int> >();
3080  fieldlen = sizes[col];
3081  valuelen = min(2 * sizeof(int) * val.size(), fieldlen);
3082  unsigned int cnt = 0;
3083  for (map<int, int>::iterator valit = val.begin(); valit != val.end(); ++valit) {
3084  memcpy(buf + offset + 2*sizeof(int)*cnt, &(valit->first), sizeof(int));
3085  memcpy(buf + offset + 2*sizeof(int)*cnt + sizeof(int), &(valit->second), sizeof(int));
3086  ++cnt;
3087  }
3088  memset(buf + offset + valuelen, 0, fieldlen - valuelen);
3089  break;
3090  }
3091  case VL_MAP_INT_INT: {
3092  Digest key = VLWrite<map<int, int>, VL_MAP_INT_INT>(a);
3093  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
3094  break;
3095  }
3096  case MAP_INT_DOUBLE: {
3097  map<int, double> val = a->cast<map<int, double> >();
3098  size_t itemsize = sizeof(int) + sizeof(double);
3099  fieldlen = sizes[col];
3100  valuelen = min(itemsize * val.size(), fieldlen);
3101  unsigned int cnt = 0;
3102  for (map<int, double>::iterator valit = val.begin(); valit != val.end(); ++valit) {
3103  memcpy(buf + offset + itemsize*cnt, &(valit->first), sizeof(int));
3104  memcpy(buf + offset + itemsize*cnt + sizeof(int), &(valit->second),
3105  sizeof(double));
3106  ++cnt;
3107  }
3108  memset(buf + offset + valuelen, 0, fieldlen - valuelen);
3109  break;
3110  }
3111  case VL_MAP_INT_DOUBLE: {
3112  Digest key = VLWrite<map<int, double>, VL_MAP_INT_DOUBLE>(a);
3113  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
3114  break;
3115  }
3116  case MAP_INT_STRING: {
3117  map<int, string> val = a->cast<map<int, string> >();
3118  shape = shapes[col];
3119  //adjusted for testing
3120  int strlen = shape[2];
3121  fieldlen = sizeof(int) + strlen;
3122  unsigned int cnt = 0;
3123  map<int, string>::iterator valit = val.begin();
3124  for (; valit != val.end(); ++valit) {
3125  memcpy(buf + offset + fieldlen*cnt, &(valit->first), sizeof(int));
3126  valuelen = std::min(static_cast<int>(valit->second.size()), strlen);
3127  memcpy(buf + offset + fieldlen*cnt + sizeof(int),
3128  valit->second.c_str(), valuelen);
3129  memset(buf + offset + fieldlen*cnt + sizeof(int) + valuelen, 0,
3130  strlen - valuelen);
3131  ++cnt;
3132  }
3133  memset(buf + offset + fieldlen*cnt, 0, fieldlen * (shape[0] - cnt));
3134  break;
3135  }
3136  case MAP_INT_VL_STRING: {
3137  map<int, string> val = a->cast<map<int, string> >();
3138  Digest valhash;
3139  fieldlen = sizeof(int) + CYCLUS_SHA1_SIZE;
3140  unsigned int cnt = 0;
3141  map<int, string>::iterator valit = val.begin();
3142  for (; valit != val.end(); ++valit) {
3143  memcpy(buf + offset + fieldlen*cnt, &(valit->first), sizeof(int));
3144  valhash = VLWrite<string, VL_STRING>(valit->second);
3145  memcpy(buf + offset + fieldlen*cnt + sizeof(int), valhash.val,
3147  ++cnt;
3148  }
3149  memset(buf + offset + fieldlen*cnt, 0, fieldlen * (val.size() - cnt));
3150  break;
3151  }
3152  case VL_MAP_INT_STRING: {
3153  shape = shapes[col];
3154  size_t strlen = shape[1];
3155  map<int, string> givenval = a->cast<map<int, string> >();
3156  map<int, string> val;
3157  // ensure string is of specified length
3158  map<int, string>::iterator valit = givenval.begin();
3159  for (; valit != givenval.end(); ++valit)
3160  val[valit->first] = string(valit->second, 0, strlen);
3161  Digest key = VLWrite<map<int, string>, VL_MAP_INT_STRING>(val);
3162  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
3163  break;
3164  }
3165  case VL_MAP_INT_VL_STRING: {
3166  Digest key = VLWrite<map<int, string>, VL_MAP_INT_VL_STRING>(a);
3167  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
3168  break;
3169  }
3170  case MAP_STRING_INT: {
3171  map<string, int> val = a->cast<map<string, int> >();
3172  shape = shapes[col];
3173  int strlen = shape[1];
3174  fieldlen = sizeof(int) + strlen;
3175  unsigned int cnt = 0;
3176  map<string, int>::iterator valit = val.begin();
3177  for (; valit != val.end(); ++valit) {
3178  valuelen = std::min(static_cast<int>(valit->first.size()), strlen);
3179  memcpy(buf + offset + fieldlen*cnt, valit->first.c_str(), valuelen);
3180  memset(buf + offset + fieldlen*cnt + valuelen, 0, strlen - valuelen);
3181  memcpy(buf + offset + fieldlen*cnt + strlen, &(valit->second), sizeof(int));
3182  ++cnt;
3183  }
3184  memset(buf + offset + fieldlen*cnt, 0, fieldlen * (shape[0] - cnt));
3185  break;
3186  }
3187  case MAP_VL_STRING_INT: {
3188  map<string, int> val = a->cast<map<string, int> >();
3189  Digest keyhash;
3190  fieldlen = sizeof(int) + CYCLUS_SHA1_SIZE;
3191  unsigned int cnt = 0;
3192  map<string, int>::iterator valit = val.begin();
3193  for (; valit != val.end(); ++valit) {
3194  keyhash = VLWrite<string, VL_STRING>(valit->first);
3195  memcpy(buf + offset + fieldlen*cnt, keyhash.val, CYCLUS_SHA1_SIZE);
3196  memcpy(buf + offset + fieldlen*cnt + CYCLUS_SHA1_SIZE, &(valit->second),
3197  sizeof(int));
3198  ++cnt;
3199  }
3200  memset(buf + offset + fieldlen*cnt, 0, fieldlen * (val.size() - cnt));
3201  break;
3202  }
3203  case VL_MAP_STRING_INT: {
3204  shape = shapes[col];
3205  size_t strlen = shape[1];
3206  map<string, int> givenval = a->cast<map<string, int> >();
3207  map<string, int> val;
3208  // ensure string is of specified length
3209  map<string, int>::iterator valit = givenval.begin();
3210  for (; valit != givenval.end(); ++valit)
3211  val[string(valit->first, 0, strlen)] = valit->second;
3212  Digest key = VLWrite<map<string, int>, VL_MAP_STRING_INT>(val);
3213  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
3214  break;
3215  }
3216  case VL_MAP_VL_STRING_INT: {
3217  Digest key = VLWrite<map<string, int>, VL_MAP_VL_STRING_INT>(a);
3218  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
3219  break;
3220  }
3221  case MAP_STRING_DOUBLE: {
3222  map<string, double> val = a->cast<map<string, double> >();
3223  shape = shapes[col];
3224  int strlen = shape[1];
3225  fieldlen = sizeof(double) + strlen;
3226  unsigned int cnt = 0;
3227  map<string, double>::iterator valit = val.begin();
3228  for (; valit != val.end(); ++valit) {
3229  valuelen = std::min(static_cast<int>(valit->first.size()), strlen);
3230  memcpy(buf + offset + fieldlen*cnt, valit->first.c_str(), valuelen);
3231  memset(buf + offset + fieldlen*cnt + valuelen, 0, strlen - valuelen);
3232  memcpy(buf + offset + fieldlen*cnt + strlen, &(valit->second),
3233  sizeof(double));
3234  ++cnt;
3235  }
3236  memset(buf + offset + fieldlen*cnt, 0, fieldlen * (shape[0] - cnt));
3237  break;
3238  }
3239  case VL_MAP_STRING_DOUBLE: {
3240  shape = shapes[col];
3241  size_t strlen = shape[1];
3242  map<string, double> givenval = a->cast<map<string, double> >();
3243  map<string, double> val;
3244  // ensure string is of specified length
3245  map<string, double>::iterator valit = givenval.begin();
3246  for (; valit != givenval.end(); ++valit)
3247  val[string(valit->first, 0, strlen)] = valit->second;
3248  Digest key = VLWrite<map<string, double>, VL_MAP_STRING_DOUBLE>(val);
3249  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
3250  break;
3251  }
3252  case MAP_STRING_STRING: {
3253  map<string, string> val = a->cast<map<string, string> >();
3254  shape = shapes[col];
3255  int keylen = shape[1];
3256  int vallen = shape[2];
3257  fieldlen = keylen + vallen;
3258  unsigned int cnt = 0;
3259  int truekeylen;
3260  int truevallen;
3261  map<string, string>::iterator valit = val.begin();
3262  for (; valit != val.end(); ++valit) {
3263  truekeylen = std::min(static_cast<int>(valit->first.size()), keylen);
3264  memcpy(buf + offset + fieldlen*cnt, valit->first.c_str(), truekeylen);
3265  memset(buf + offset + fieldlen*cnt + truekeylen, 0, keylen - truekeylen);
3266  truevallen = std::min(static_cast<int>(valit->second.size()), vallen);
3267  memcpy(buf + offset + fieldlen*cnt + keylen, valit->second.c_str(),
3268  truevallen);
3269  memset(buf + offset + fieldlen*cnt + keylen + truekeylen, 0,
3270  vallen - truevallen);
3271  ++cnt;
3272  }
3273  memset(buf + offset + fieldlen*cnt, 0, fieldlen * (shape[0] - cnt));
3274  break;
3275  }
3276  case VL_MAP_STRING_STRING: {
3277  shape = shapes[col];
3278  size_t keylen = shape[1];
3279  size_t vallen = shape[2];
3280  map<string, string> givenval = a->cast<map<string, string> >();
3281  map<string, string> val;
3282  // ensure strings of specified length
3283  map<string, string>::iterator valit = givenval.begin();
3284  for (; valit != givenval.end(); ++valit)
3285  val[string(valit->first, 0, keylen)] = string(valit->second, 0, vallen);
3286  Digest key = VLWrite<map<string, string>, VL_MAP_STRING_STRING>(val);
3287  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
3288  break;
3289  }
3290  case MAP_STRING_VL_STRING: {
3291  map<string, string> val = a->cast<map<string, string> >();
3292  Digest valhash;
3293  shape = shapes[col];
3294  size_t keylen = shape[1];
3295  fieldlen = CYCLUS_SHA1_SIZE + keylen;
3296  int truekeylen;
3297  unsigned int cnt = 0;
3298  map<string, string>::iterator valit = val.begin();
3299  for (; valit != val.end(); ++valit) {
3300  truekeylen = std::min(valit->first.size(), keylen);
3301  memcpy(buf + offset + fieldlen*cnt, valit->first.c_str(), truekeylen);
3302  memset(buf + offset + fieldlen*cnt + truekeylen, 0, keylen - truekeylen);
3303  valhash = VLWrite<string, VL_STRING>(valit->second);
3304  memcpy(buf + offset + fieldlen*cnt + keylen, valhash.val, CYCLUS_SHA1_SIZE);
3305  ++cnt;
3306  }
3307  memset(buf + offset + fieldlen*cnt, 0, fieldlen * (val.size() - cnt));
3308  break;
3309  }
3310  case VL_MAP_STRING_VL_STRING: {
3311  Digest key = VLWrite<map<string, string>, VL_MAP_STRING_VL_STRING>(a);
3312  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
3313  break;
3314  }
3315  case MAP_VL_STRING_DOUBLE: {
3316  map<string, double> val = a->cast<map<string, double> >();
3317  Digest keyhash;
3318  fieldlen = sizeof(double) + CYCLUS_SHA1_SIZE;
3319  unsigned int cnt = 0;
3320  map<string, double>::iterator valit = val.begin();
3321  for (; valit != val.end(); ++valit) {
3322  keyhash = VLWrite<string, VL_STRING>(valit->first);
3323  memcpy(buf + offset + fieldlen*cnt, keyhash.val, CYCLUS_SHA1_SIZE);
3324  memcpy(buf + offset + fieldlen*cnt + CYCLUS_SHA1_SIZE, &(valit->second),
3325  sizeof(double));
3326  ++cnt;
3327  }
3328  memset(buf + offset + fieldlen*cnt, 0, fieldlen * (val.size() - cnt));
3329  break;
3330  }
3331  case VL_MAP_VL_STRING_DOUBLE: {
3332  Digest key = VLWrite<map<string, double>, VL_MAP_VL_STRING_DOUBLE>(a);
3333  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
3334  break;
3335  }
3336  case MAP_VL_STRING_STRING: {
3337  map<string, string> val = a->cast<map<string, string> >();
3338  Digest keyhash;
3339  shape = shapes[col];
3340  size_t vallen = shape[2];
3341  fieldlen = CYCLUS_SHA1_SIZE + vallen;
3342  int truevallen;
3343  unsigned int cnt = 0;
3344  map<string, string>::iterator valit = val.begin();
3345  for (; valit != val.end(); ++valit) {
3346  keyhash = VLWrite<string, VL_STRING>(valit->first);
3347  memcpy(buf + offset + fieldlen*cnt, keyhash.val, CYCLUS_SHA1_SIZE);
3348  truevallen = std::min(valit->second.size(), vallen);
3349  memcpy(buf + offset + fieldlen*cnt + CYCLUS_SHA1_SIZE, valit->second.c_str(), truevallen);
3350  memset(buf + offset + fieldlen*cnt + CYCLUS_SHA1_SIZE + truevallen, 0,
3351  vallen - truevallen);
3352  ++cnt;
3353  }
3354  memset(buf + offset + fieldlen*cnt, 0, fieldlen * (val.size() - cnt));
3355  break;
3356  }
3357  case VL_MAP_VL_STRING_STRING: {
3358  Digest key = VLWrite<map<string, string>, VL_MAP_VL_STRING_STRING>(a);
3359  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
3360  break;
3361  }
3362  case MAP_VL_STRING_VL_STRING: {
3363  map<string, string> val = a->cast<map<string, string> >();
3364  Digest keyhash;
3365  Digest valhash;
3366  shape = shapes[col];
3367  fieldlen = 2 * CYCLUS_SHA1_SIZE;
3368  unsigned int cnt = 0;
3369  map<string, string>::iterator valit = val.begin();
3370  for (; valit != val.end(); ++valit) {
3371  keyhash = VLWrite<string, VL_STRING>(valit->first);
3372  memcpy(buf + offset + fieldlen*cnt, keyhash.val, CYCLUS_SHA1_SIZE);
3373  valhash = VLWrite<string, VL_STRING>(valit->second);
3374  memcpy(buf + offset + fieldlen*cnt + CYCLUS_SHA1_SIZE, valhash.val,
3376  ++cnt;
3377  }
3378  memset(buf + offset + fieldlen*cnt, 0, fieldlen * (val.size() - cnt));
3379  break;
3380  }
3382  Digest key = VLWrite<map<string, string>, VL_MAP_VL_STRING_VL_STRING>(a);
3383  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
3384  break;
3385  }
3387  map<pair<int, string>, double> val = \
3388  a->cast<map<pair<int, string>, double> >();
3389  shape = shapes[col];
3390  int strlen = shape[3];
3391  fieldlen = sizeof(int) + strlen + sizeof(double);
3392  unsigned int cnt = 0;
3393  map<pair<int, string>, double>::iterator valit = val.begin();
3394  for (; valit != val.end(); ++valit) {
3395  valuelen = std::min(static_cast<int>(valit->first.second.size()), strlen);
3396  memcpy(buf + offset + fieldlen*cnt, &(valit->first.first), sizeof(int));
3397  memcpy(buf + offset + fieldlen*cnt + sizeof(int),
3398  valit->first.second.c_str(), valuelen);
3399  memset(buf + offset + fieldlen*cnt + sizeof(int) + valuelen, 0,
3400  strlen - valuelen);
3401  memcpy(buf + offset + fieldlen*cnt + sizeof(int) + strlen, &(valit->second),
3402  sizeof(double));
3403  ++cnt;
3404  }
3405  memset(buf + offset + fieldlen*cnt, 0, fieldlen * (shape[0] - cnt));
3406  break;
3407  }
3409  shape = shapes[col];
3410  size_t strlen = shape[3];
3411  map<pair<int, string>, double> givenval = \
3412  a->cast<map<pair<int, string>, double> >();
3413  map<pair<int, string>, double> val;
3414  // ensure string is of specified length
3415  map<pair<int, string>, double>::iterator valit = givenval.begin();
3416  for (; valit != givenval.end(); ++valit)
3417  val[std::make_pair(valit->first.first,
3418  string(valit->first.second, 0, strlen))] = valit->second;
3419  Digest key = VLWrite<map<pair<int, string>, double>,
3421  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
3422  break;
3423  }
3425  map<pair<int, string>, double> val = \
3426  a->cast<map<pair<int, string>, double> >();
3427  Digest keyhash;
3428  fieldlen = sizeof(int) + CYCLUS_SHA1_SIZE + sizeof(double);
3429  unsigned int cnt = 0;
3430  map<pair<int, string>, double>::iterator valit = val.begin();
3431  for (; valit != val.end(); ++valit) {
3432  keyhash = VLWrite<string, VL_STRING>(valit->first.second);
3433  memcpy(buf + offset + fieldlen*cnt, &(valit->first.first), sizeof(int));
3434  memcpy(buf + offset + fieldlen*cnt + sizeof(int), keyhash.val,
3436  memcpy(buf + offset + fieldlen*cnt + sizeof(int) + CYCLUS_SHA1_SIZE,
3437  &(valit->second), sizeof(double));
3438  ++cnt;
3439  }
3440  memset(buf + offset + fieldlen*cnt, 0, fieldlen * (val.size() - cnt));
3441  break;
3442  }
3444  Digest key = VLWrite<map<pair<int, string>, double>,
3446  memcpy(buf + offset, key.val, CYCLUS_SHA1_SIZE);
3447  break;
3448  }
3449  default: {
3450  throw ValueError("attempted to retrieve unsupported HDF5 backend type");
3451  }
3452  }
3453  offset += sizes[col];
3454  }
3455  }
3456 }
3457 
3458 template <typename T, DbTypes U>
3459 T Hdf5Back::VLRead(const char* rawkey) {
3460  // key is used as offset
3461  Digest key;
3462  memcpy(key.val, rawkey, CYCLUS_SHA1_SIZE);
3463  const std::vector<hsize_t> idx = key.cast<hsize_t>();
3464  hid_t dset = VLDataset(U, false);
3465  hid_t dspace = H5Dget_space(dset);
3466  hid_t mspace = H5Screate_simple(CYCLUS_SHA1_NINT, vlchunk_, NULL);
3467  herr_t status = H5Sselect_hyperslab(dspace, H5S_SELECT_SET, (const hsize_t*) &idx[0],
3468  NULL, vlchunk_, NULL);
3469  if (status < 0)
3470  throw IOError("could not select hyperslab of value array for reading "
3471  "in the database '" + path_ + "'.");
3472  hvl_t buf;
3473  status = H5Dread(dset, vldts_[U], mspace, dspace, H5P_DEFAULT, &buf);
3474  if (status < 0) {
3475  std::stringstream ss;
3476  ss << U;
3477  throw IOError("failed to read in variable length data "
3478  "in the database '" + path_ + "' (type id " + ss.str() +
3479  ").");
3480  }
3481  T val = VLBufToVal<T>(buf);
3482  status = H5Dvlen_reclaim(vldts_[U], mspace, H5P_DEFAULT, &buf);
3483  if (status < 0)
3484  throw IOError("failed to reclaim variable length data space "
3485  "in the database '" + path_ + "'.");
3486  H5Sclose(mspace);
3487  H5Sclose(dspace);
3488  return val;
3489 }
3490 
3491 
3492 hid_t Hdf5Back::VLDataset(DbTypes dbtype, bool forkeys) {
3493  std::string name;
3494  switch (dbtype) {
3495  case VL_STRING: {
3496  name = "String";
3497  break;
3498  }
3499  case BLOB: {
3500  name = "Blob";
3501  break;
3502  }
3503  case VL_VECTOR_INT: {
3504  name = "VectorInt";
3505  break;
3506  }
3507  case VL_VECTOR_FLOAT: {
3508  name = "VectorFloat";
3509  break;
3510  }
3511  case VL_VECTOR_DOUBLE: {
3512  name = "VectorDouble";
3513  break;
3514  }
3515  case VL_VECTOR_STRING:
3516  case VL_VECTOR_VL_STRING: {
3517  name = "VectorString";
3518  break;
3519  }
3520  case VL_SET_INT: {
3521  name = "SetInt";
3522  break;
3523  }
3524  case VL_SET_STRING:
3525  case VL_SET_VL_STRING: {
3526  name = "SetString";
3527  break;
3528  }
3529  case VL_LIST_INT: {
3530  name = "ListInt";
3531  break;
3532  }
3533  case VL_LIST_STRING:
3534  case VL_LIST_VL_STRING: {
3535  name = "ListString";
3536  break;
3537  }
3538  case VL_MAP_INT_INT: {
3539  name = "MapIntInt";
3540  break;
3541  }
3542  case VL_MAP_INT_DOUBLE: {
3543  name = "MapIntDouble";
3544  break;
3545  }
3546  case VL_MAP_INT_STRING:
3547  case VL_MAP_INT_VL_STRING: {
3548  name = "MapIntString";
3549  break;
3550  }
3551  case VL_MAP_STRING_INT:
3552  case VL_MAP_VL_STRING_INT: {
3553  name = "MapStringInt";
3554  break;
3555  }
3556  case VL_MAP_STRING_DOUBLE:
3557  case VL_MAP_VL_STRING_DOUBLE: {
3558  name = "MapStringDouble";
3559  break;
3560  }
3561  case VL_MAP_STRING_STRING:
3565  name = "MapStringString";
3566  break;
3567  }
3570  name = "MapPairIntStringDouble";
3571  break;
3572  }
3573  default: {
3574  throw IOError("could not determine variable length dataset name.");
3575  break;
3576  }
3577  }
3578  name += forkeys ? "Keys" : "Vals";
3579 
3580  // already opened
3581  if (vldatasets_.count(name) > 0)
3582  return vldatasets_[name];
3583 
3584  // already in db
3585  hid_t dt;
3586  hid_t dset;
3587  hid_t dspace;
3588  herr_t status;
3589  if (H5Lexists(file_, name.c_str(), H5P_DEFAULT)) {
3590  dset = H5Dopen2(file_, name.c_str(), H5P_DEFAULT);
3591  if (forkeys) {
3592  // read in existing keys to vlkeys_
3593  dspace = H5Dget_space(dset);
3594  unsigned int nkeys = H5Sget_simple_extent_npoints(dspace);
3595  char* buf = new char[CYCLUS_SHA1_SIZE * nkeys];
3596  status = H5Dread(dset, sha1_type_, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf);
3597  if (status < 0)
3598  throw IOError("failed to read in keys for " + name);
3599  for (int n = 0; n < nkeys; ++n) {
3600  Digest d = Digest();
3601  memcpy(d.val, buf + (n * CYCLUS_SHA1_SIZE), CYCLUS_SHA1_SIZE);
3602  vlkeys_[dbtype].insert(d);
3603  }
3604  H5Sclose(dspace);
3605  delete[] buf;
3606  } else {
3607  if (vldts_.count(dbtype) == 0) {
3608  dt = H5Dget_type(dset);
3609  if (dt < 0)
3610  throw IOError("failed to read in HDF5 datatype for " + name);
3611  vldts_[dbtype] = dt;
3612  }
3613  }
3614  vldatasets_[name] = dset;
3615  return dset;
3616  }
3617 
3618  // doesn't exist at all
3619  hid_t prop;
3620  if (forkeys) {
3621  hsize_t dims[1] = {0};
3622  hsize_t maxdims[1] = {H5S_UNLIMITED};
3623  hsize_t chunkdims[1] = {512}; // this is a 10 kb chunksize
3624  dt = sha1_type_;
3625  dspace = H5Screate_simple(1, dims, maxdims);
3626  prop = H5Pcreate(H5P_DATASET_CREATE);
3627  status = H5Pset_chunk(prop, 1, chunkdims);
3628  if (status < 0)
3629  throw IOError("could not create HDF5 array " + name);
3630  } else {
3631  hsize_t dims[CYCLUS_SHA1_NINT] = {UINT_MAX, UINT_MAX, UINT_MAX, UINT_MAX, UINT_MAX};
3632  hsize_t chunkdims[CYCLUS_SHA1_NINT] = {1, 1, 1, 1, 1}; // this is a single element
3633  dt = vldts_[dbtype];
3634  dspace = H5Screate_simple(CYCLUS_SHA1_NINT, dims, dims);
3635  prop = H5Pcreate(H5P_DATASET_CREATE);
3636  status = H5Pset_chunk(prop, CYCLUS_SHA1_NINT, chunkdims);
3637  if (status < 0)
3638  throw IOError("could not create HDF5 array " + name);
3639  }
3640  dset = H5Dcreate2(file_, name.c_str(), dt, dspace, H5P_DEFAULT, prop, H5P_DEFAULT);
3641  vldatasets_[name] = dset;
3642  return dset;
3643 }
3644 
3645 void Hdf5Back::AppendVLKey(hid_t dset, DbTypes dbtype, const Digest& key) {
3646  hid_t dspace = H5Dget_space(dset);
3647  hsize_t origlen = H5Sget_simple_extent_npoints(dspace);
3648  hsize_t newlen[1] = {origlen + 1};
3649  hsize_t offset[1] = {origlen};
3650  hsize_t extent[1] = {1};
3651  hid_t mspace = H5Screate_simple(1, extent, NULL);
3652  herr_t status = H5Dextend(dset, newlen);
3653  if (status < 0)
3654  throw IOError("could not resize key array in the database '" + path_ + "'.");
3655  dspace = H5Dget_space(dset);
3656  status = H5Sselect_hyperslab(dspace, H5S_SELECT_SET, offset, NULL, extent, NULL);
3657  if (status < 0)
3658  throw IOError("could not select hyperslab of key array "
3659  "in the database '" + path_ + "'.");
3660  status = H5Dwrite(dset, sha1_type_, mspace, dspace, H5P_DEFAULT, key.val);
3661  if (status < 0)
3662  throw IOError("could not write digest to key array "
3663  "in the database '" + path_ + "'.");
3664  H5Sclose(mspace);
3665  H5Sclose(dspace);
3666  vlkeys_[dbtype].insert(key);
3667 }
3668 
3669 void Hdf5Back::InsertVLVal(hid_t dset, DbTypes dbtype, const Digest& key,
3670  const std::string& val) {
3671  hid_t dspace = H5Dget_space(dset);
3672  hsize_t extent[CYCLUS_SHA1_NINT] = {1, 1, 1, 1, 1};
3673  hid_t mspace = H5Screate_simple(CYCLUS_SHA1_NINT, extent, NULL);
3674  const std::vector<hsize_t> idx = key.cast<hsize_t>();
3675  herr_t status = H5Sselect_hyperslab(dspace, H5S_SELECT_SET, (const hsize_t*) &idx[0],
3676  NULL, extent, NULL);
3677  if (status < 0)
3678  throw IOError("could not select hyperslab of value array "
3679  "in the database '" + path_ + "'.");
3680  const char* buf[1] = {val.c_str()};
3681  status = H5Dwrite(dset, vldts_[dbtype], mspace, dspace, H5P_DEFAULT, buf);
3682  if (status < 0)
3683  throw IOError("could not write string to value array "
3684  "in the database '" + path_ + "'.");
3685  H5Sclose(mspace);
3686  H5Sclose(dspace);
3687 }
3688 
3689 void Hdf5Back::InsertVLVal(hid_t dset, DbTypes dbtype, const Digest& key,
3690  hvl_t buf) {
3691  hid_t dspace = H5Dget_space(dset);
3692  hsize_t extent[CYCLUS_SHA1_NINT] = {1, 1, 1, 1, 1};
3693  hid_t mspace = H5Screate_simple(CYCLUS_SHA1_NINT, extent, NULL);
3694  const std::vector<hsize_t> idx = key.cast<hsize_t>();
3695  herr_t status = H5Sselect_hyperslab(dspace, H5S_SELECT_SET, (const hsize_t*) &idx[0],
3696  NULL, extent, NULL);
3697  if (status < 0)
3698  throw IOError("could not select hyperslab of value array "
3699  "in the database '" + path_ + "'.");
3700  status = H5Dwrite(dset, vldts_[dbtype], mspace, dspace, H5P_DEFAULT, &buf);
3701  if (status < 0)
3702  throw IOError("could not write variable length data to value array "
3703  "in the database '" + path_ + "'.");
3704  status = H5Dvlen_reclaim(vldts_[dbtype], mspace, H5P_DEFAULT, &buf);
3705  if (status < 0)
3706  throw IOError("could not free variable length buffer "
3707  "in the database '" + path_ + "'.");
3708  H5Sclose(mspace);
3709  H5Sclose(dspace);
3710 }
3711 
3712 hvl_t Hdf5Back::VLValToBuf(const std::vector<int>& x) {
3713  hvl_t buf;
3714  buf.len = x.size();
3715  size_t nbytes = buf.len * sizeof(int);
3716  buf.p = new char[nbytes];
3717  memcpy(buf.p, &x[0], nbytes);
3718  return buf;
3719 }
3720 
3721 template <>
3722 std::vector<int> Hdf5Back::VLBufToVal<std::vector<int> >(const hvl_t& buf) {
3723  std::vector<int> x = std::vector<int>(buf.len);
3724  memcpy(&x[0], buf.p, buf.len * sizeof(int));
3725  return x;
3726 }
3727 
3728 hvl_t Hdf5Back::VLValToBuf(const std::vector<float>& x) {
3729  hvl_t buf;
3730  buf.len = x.size();
3731  size_t nbytes = buf.len * sizeof(float);
3732  buf.p = new char[nbytes];
3733  memcpy(buf.p, &x[0], nbytes);
3734  return buf;
3735 }
3736 
3737 template <>
3738 std::vector<float> Hdf5Back::VLBufToVal<std::vector<float> >(const hvl_t& buf) {
3739  std::vector<float> x = std::vector<float>(buf.len);
3740  memcpy(&x[0], buf.p, buf.len * sizeof(float));
3741  return x;
3742 }
3743 
3744 hvl_t Hdf5Back::VLValToBuf(const std::vector<double>& x) {
3745  hvl_t buf;
3746  buf.len = x.size();
3747  size_t nbytes = buf.len * sizeof(double);
3748  buf.p = new char[nbytes];
3749  memcpy(buf.p, &x[0], nbytes);
3750  return buf;
3751 }
3752 
3753 template <>
3754 std::vector<double> Hdf5Back::VLBufToVal<std::vector<double> >(const hvl_t& buf) {
3755  std::vector<double> x = std::vector<double>(buf.len);
3756  memcpy(&x[0], buf.p, buf.len * sizeof(double));
3757  return x;
3758 }
3759 
3760 hvl_t Hdf5Back::VLValToBuf(const std::vector<std::string>& x) {
3761  // VL_VECTOR_STRING implemented as VL_VECTOR_VL_STRING
3762  hvl_t buf;
3763  Digest key;
3764  buf.len = x.size();
3765  size_t nbytes = CYCLUS_SHA1_SIZE * buf.len;
3766  buf.p = new char[nbytes];
3767  for (unsigned int i = 0; i < buf.len; ++i) {
3768  key = VLWrite<std::string, VL_STRING>(x[i]);
3769  memcpy((char *) buf.p + CYCLUS_SHA1_SIZE*i, key.val, CYCLUS_SHA1_SIZE);
3770  }
3771  return buf;
3772 }
3773 
3774 template <>
3775 std::vector<std::string> Hdf5Back::VLBufToVal<std::vector<std::string> >(const hvl_t& buf) {
3776  using std::string;
3777  std::vector<string> x = std::vector<string>(buf.len);
3778  for (unsigned int i = 0; i < buf.len; ++i)
3779  x[i] = VLRead<string, VL_STRING>((char *) buf.p + CYCLUS_SHA1_SIZE*i);
3780  return x;
3781 }
3782 
3783 hvl_t Hdf5Back::VLValToBuf(const std::set<int>& x) {
3784  hvl_t buf;
3785  buf.len = x.size();
3786  size_t nbytes = buf.len * sizeof(int);
3787  buf.p = new char[nbytes];
3788  unsigned int cnt = 0;
3789  std::set<int>::iterator it = x.begin();
3790  for (; it != x.end(); ++it) {
3791  memcpy((char *) buf.p + cnt*sizeof(int), &(*it), sizeof(int));
3792  ++cnt;
3793  }
3794  return buf;
3795 }
3796 
3797 template <>
3798 std::set<int> Hdf5Back::VLBufToVal<std::set<int> >(const hvl_t& buf) {
3799  int* xraw = reinterpret_cast<int*>(buf.p);
3800  std::set<int> x = std::set<int>(xraw, xraw+buf.len);
3801  return x;
3802 }
3803 
3804 hvl_t Hdf5Back::VLValToBuf(const std::set<std::string>& x) {
3805  // VL_SET_STRING implemented as VL_SET_VL_STRING
3806  hvl_t buf;
3807  Digest key;
3808  buf.len = x.size();
3809  size_t nbytes = CYCLUS_SHA1_SIZE * buf.len;
3810  buf.p = new char[nbytes];
3811  unsigned int i = 0;
3812  std::set<std::string>::iterator it = x.begin();
3813  for (; it != x.end(); ++it) {
3814  key = VLWrite<std::string, VL_STRING>(*it);
3815  memcpy((char *) buf.p + CYCLUS_SHA1_SIZE*i, key.val, CYCLUS_SHA1_SIZE);
3816  ++i;
3817  }
3818  return buf;
3819 }
3820 
3821 template <>
3822 std::set<std::string> Hdf5Back::VLBufToVal<std::set<std::string> >(const hvl_t& buf) {
3823  using std::string;
3824  std::set<string> x;
3825  for (unsigned int i = 0; i < buf.len; ++i)
3826  x.insert(VLRead<string, VL_STRING>((char *) buf.p + CYCLUS_SHA1_SIZE*i));
3827  return x;
3828 }
3829 
3830 hvl_t Hdf5Back::VLValToBuf(const std::list<int>& x) {
3831  hvl_t buf;
3832  buf.len = x.size();
3833  size_t nbytes = buf.len * sizeof(int);
3834  buf.p = new char[nbytes];
3835  unsigned int cnt = 0;
3836  std::list<int>::const_iterator it = x.begin();
3837  for (; it != x.end(); ++it) {
3838  memcpy((char *) buf.p + cnt*sizeof(int), &(*it), sizeof(int));
3839  ++cnt;
3840  }
3841  return buf;
3842 }
3843 
3844 template <>
3845 std::list<int> Hdf5Back::VLBufToVal<std::list<int> >(const hvl_t& buf) {
3846  int* xraw = reinterpret_cast<int*>(buf.p);
3847  std::list<int> x = std::list<int>(xraw, xraw+buf.len);
3848  return x;
3849 }
3850 
3851 hvl_t Hdf5Back::VLValToBuf(const std::list<std::string>& x) {
3852  // VL_LIST_STRING implemented as VL_LIST_VL_STRING
3853  hvl_t buf;
3854  Digest key;
3855  buf.len = x.size();
3856  size_t nbytes = CYCLUS_SHA1_SIZE * buf.len;
3857  buf.p = new char[nbytes];
3858  unsigned int i = 0;
3859  std::list<std::string>::const_iterator it = x.begin();
3860  for (; it != x.end(); ++it) {
3861  key = VLWrite<std::string, VL_STRING>(*it);
3862  memcpy((char *) buf.p + CYCLUS_SHA1_SIZE*i, key.val, CYCLUS_SHA1_SIZE);
3863  ++i;
3864  }
3865  return buf;
3866 }
3867 
3868 template <>
3869 std::list<std::string> Hdf5Back::VLBufToVal<std::list<std::string> >(const hvl_t& buf) {
3870  using std::string;
3871  std::list<string> x;
3872  for (unsigned int i = 0; i < buf.len; ++i)
3873  x.push_back(VLRead<string, VL_STRING>((char *) buf.p + CYCLUS_SHA1_SIZE*i));
3874  return x;
3875 }
3876 
3877 hvl_t Hdf5Back::VLValToBuf(const std::map<int, int>& x) {
3878  hvl_t buf;
3879  buf.len = x.size();
3880  size_t nbytes = 2 * sizeof(int) * buf.len;
3881  buf.p = new char[nbytes];
3882  unsigned int cnt = 0;
3883  std::map<int, int>::const_iterator it = x.begin();
3884  for (; it != x.end(); ++it) {
3885  memcpy((char *) buf.p + 2*sizeof(int)*cnt, &(it->first), sizeof(int));
3886  memcpy((char *) buf.p + 2*sizeof(int)*cnt + sizeof(int), &(it->second), sizeof(int));
3887  ++cnt;
3888  }
3889  return buf;
3890 }
3891 
3892 template <>
3893 std::map<int, int> Hdf5Back::VLBufToVal<std::map<int, int> >(const hvl_t& buf) {
3894  int* xraw = reinterpret_cast<int*>(buf.p);
3895  std::map<int, int> x = std::map<int, int>();
3896  for (unsigned int i = 0; i < buf.len; ++i)
3897  x[xraw[2*i]] = xraw[2*i + 1];
3898  return x;
3899 }
3900 
3901 hvl_t Hdf5Back::VLValToBuf(const std::map<int, double>& x) {
3902  hvl_t buf;
3903  buf.len = x.size();
3904  size_t itemsize = sizeof(int) + sizeof(double);
3905  size_t nbytes = itemsize * buf.len;
3906  buf.p = new char[nbytes];
3907  unsigned int cnt = 0;
3908  std::map<int, double>::const_iterator it = x.begin();
3909  for (; it != x.end(); ++it) {
3910  memcpy((char *) buf.p + itemsize*cnt, &(it->first), sizeof(int));
3911  memcpy((char *) buf.p + itemsize*cnt + sizeof(int), &(it->second), sizeof(double));
3912  ++cnt;
3913  }
3914  return buf;
3915 }
3916 
3917 template <>
3918 std::map<int, double> Hdf5Back::VLBufToVal<std::map<int, double> >(const hvl_t& buf) {
3919  std::map<int, double> x;
3920  char * p = reinterpret_cast<char*>(buf.p);
3921  size_t itemsize = sizeof(int) + sizeof(double);
3922  for (unsigned int i = 0; i < buf.len; ++i)
3923  x[*reinterpret_cast<int*>(p + itemsize*i)] = \
3924  *reinterpret_cast<double*>(p + itemsize*i + sizeof(int));
3925  return x;
3926 }
3927 
3928 hvl_t Hdf5Back::VLValToBuf(const std::map<int, std::string>& x) {
3929  // VL_MAP_INT_STRING implemented as VL_MAP_INT_VL_STRING
3930  hvl_t buf;
3931  Digest valhash;
3932  buf.len = x.size();
3933  size_t itemsize = sizeof(int) + CYCLUS_SHA1_SIZE;
3934  size_t nbytes = itemsize * buf.len;
3935  buf.p = new char[nbytes];
3936  unsigned int i = 0;
3937  std::map<int, std::string>::const_iterator it = x.begin();
3938  for (; it != x.end(); ++it) {
3939  memcpy((char *) buf.p + itemsize*i, &(it->first), sizeof(int));
3940  valhash = VLWrite<std::string, VL_STRING>(it->second);
3941  memcpy((char *) buf.p + itemsize*i + sizeof(int), valhash.val, CYCLUS_SHA1_SIZE);
3942  ++i;
3943  }
3944  return buf;
3945 }
3946 
3947 template <> std::map<int, std::string>
3948 Hdf5Back::VLBufToVal<std::map<int, std::string> >(const hvl_t& buf) {
3949  using std::string;
3950  std::map<int, string> x;
3951  char * p = reinterpret_cast<char*>(buf.p);
3952  size_t itemsize = sizeof(int) + CYCLUS_SHA1_SIZE;
3953  for (unsigned int i = 0; i < buf.len; ++i)
3954  x[*reinterpret_cast<int*>(p + itemsize*i)] = \
3955  VLRead<string, VL_STRING>(p + itemsize*i + sizeof(int));
3956  return x;
3957 }
3958 
3959 hvl_t Hdf5Back::VLValToBuf(const std::map<std::string, int>& x) {
3960  // VL_MAP_STRING_INT implemented as VL_MAP_VL_STRING_INT
3961  hvl_t buf;
3962  Digest keyhash;
3963  buf.len = x.size();
3964  size_t itemsize = sizeof(int) + CYCLUS_SHA1_SIZE;
3965  size_t nbytes = itemsize * buf.len;
3966  buf.p = new char[nbytes];
3967  unsigned int i = 0;
3968  std::map<std::string, int>::const_iterator it = x.begin();
3969  for (; it != x.end(); ++it) {
3970  keyhash = VLWrite<std::string, VL_STRING>(it->first);
3971  memcpy((char *) buf.p + itemsize*i, keyhash.val, CYCLUS_SHA1_SIZE);
3972  memcpy((char *) buf.p + itemsize*i + CYCLUS_SHA1_SIZE, &(it->second), sizeof(int));
3973  ++i;
3974  }
3975  return buf;
3976 }
3977 
3978 template <> std::map<std::string, int>
3979 Hdf5Back::VLBufToVal<std::map<std::string, int> >(const hvl_t& buf) {
3980  using std::string;
3981  std::map<string, int> x;
3982  char * p = reinterpret_cast<char*>(buf.p);
3983  size_t itemsize = sizeof(int) + CYCLUS_SHA1_SIZE;
3984  for (unsigned int i = 0; i < buf.len; ++i)
3985  x[VLRead<string, VL_STRING>(p + itemsize*i)] = \
3986  *reinterpret_cast<int*>(p + itemsize*i + CYCLUS_SHA1_SIZE);
3987  return x;
3988 }
3989 
3990 hvl_t Hdf5Back::VLValToBuf(const std::map<std::string, double>& x) {
3991  // VL_MAP_STRING_DOUBLE implemented as VL_MAP_VL_STRING_DOUBLE
3992  hvl_t buf;
3993  Digest keyhash;
3994  buf.len = x.size();
3995  size_t itemsize = sizeof(double) + CYCLUS_SHA1_SIZE;
3996  size_t nbytes = itemsize * buf.len;
3997  buf.p = new char[nbytes];
3998  unsigned int i = 0;
3999  std::map<std::string, double>::const_iterator it = x.begin();
4000  for (; it != x.end(); ++it) {
4001  keyhash = VLWrite<std::string, VL_STRING>(it->first);
4002  memcpy((char *) buf.p + itemsize*i, keyhash.val, CYCLUS_SHA1_SIZE);
4003  memcpy((char *) buf.p + itemsize*i + CYCLUS_SHA1_SIZE, &(it->second),
4004  sizeof(double));
4005  ++i;
4006  }
4007  return buf;
4008 }
4009 
4010 template <> std::map<std::string, double>
4011 Hdf5Back::VLBufToVal<std::map<std::string, double> >(const hvl_t& buf) {
4012  using std::string;
4013  std::map<string, double> x;
4014  char * p = reinterpret_cast<char*>(buf.p);
4015  size_t itemsize = sizeof(double) + CYCLUS_SHA1_SIZE;
4016  for (unsigned int i = 0; i < buf.len; ++i)
4017  x[VLRead<string, VL_STRING>(p + itemsize*i)] = \
4018  *reinterpret_cast<double*>(p + itemsize*i + CYCLUS_SHA1_SIZE);
4019  return x;
4020 }
4021 
4022 hvl_t Hdf5Back::VLValToBuf(const std::map<std::string, std::string>& x) {
4023  // VL_MAP_STRING_STRING, VL_MAP_STRING_VL_STRING, and VL_MAP_VL_STRING_STRING
4024  // implemented as VL_MAP_VL_STRING_VL_STRING
4025  hvl_t buf;
4026  Digest keyhash;
4027  Digest valhash;
4028  buf.len = x.size();
4029  size_t itemsize = 2*CYCLUS_SHA1_SIZE;
4030  size_t nbytes = itemsize * buf.len;
4031  buf.p = new char[nbytes];
4032  unsigned int i = 0;
4033  std::map<std::string, std::string>::const_iterator it = x.begin();
4034  for (; it != x.end(); ++it) {
4035  keyhash = VLWrite<std::string, VL_STRING>(it->first);
4036  memcpy((char *) buf.p + itemsize*i, keyhash.val, CYCLUS_SHA1_SIZE);
4037  valhash = VLWrite<std::string, VL_STRING>(it->second);
4038  memcpy((char *) buf.p + itemsize*i + CYCLUS_SHA1_SIZE, valhash.val,
4040  ++i;
4041  }
4042  return buf;
4043 }
4044 
4045 template <> std::map<std::string, std::string>
4046 Hdf5Back::VLBufToVal<std::map<std::string, std::string> >(const hvl_t& buf) {
4047  using std::string;
4048  std::map<string, string> x;
4049  char * p = reinterpret_cast<char*>(buf.p);
4050  size_t itemsize = 2 * CYCLUS_SHA1_SIZE;
4051  for (unsigned int i = 0; i < buf.len; ++i)
4052  x[VLRead<string, VL_STRING>(p + itemsize*i)] = \
4053  VLRead<string, VL_STRING>(p + itemsize*i + CYCLUS_SHA1_SIZE);
4054  return x;
4055 }
4056 
4057 
4058 hvl_t Hdf5Back::VLValToBuf(const std::map<std::pair<int, std::string>, double>& x) {
4059  // VL_MAP_PAIR_INT_STRING_DOUBLE implemented as VL_MAP_PAIR_INT_VL_STRING_DOUBLE
4060  hvl_t buf;
4061  Digest keyhash;
4062  buf.len = x.size();
4063  size_t itemsize = sizeof(int) + CYCLUS_SHA1_SIZE + sizeof(double);
4064  size_t nbytes = itemsize * buf.len;
4065  buf.p = new char[nbytes];
4066  unsigned int i = 0;
4067  std::map<std::pair<int, std::string>, double>::const_iterator it = x.begin();
4068  for (; it != x.end(); ++it) {
4069  keyhash = VLWrite<std::string, VL_STRING>(it->first.second);
4070  memcpy((char *) buf.p + itemsize*i, &(it->first.first), sizeof(int));
4071  memcpy((char *) buf.p + itemsize*i + sizeof(int), keyhash.val, CYCLUS_SHA1_SIZE);
4072  memcpy((char *) buf.p + itemsize*i + sizeof(int) + CYCLUS_SHA1_SIZE, &(it->second),
4073  sizeof(double));
4074  ++i;
4075  }
4076  return buf;
4077 }
4078 
4079 template <> std::map<std::pair<int, std::string>, double>
4080 Hdf5Back::VLBufToVal<std::map<std::pair<int, std::string>, double> >(const hvl_t& buf) {
4081  using std::string;
4082  using std::pair;
4083  std::map<pair<int, string>, double> x;
4084  char * p = reinterpret_cast<char*>(buf.p);
4085  size_t itemsize = sizeof(int) + CYCLUS_SHA1_SIZE + sizeof(double);
4086  for (unsigned int i = 0; i < buf.len; ++i)
4087  x[std::make_pair(*reinterpret_cast<int*>(p + itemsize*i),
4088  VLRead<string, VL_STRING>(p + itemsize*i + sizeof(int)))] = \
4089  *reinterpret_cast<double*>(p + itemsize*i + sizeof(int) + CYCLUS_SHA1_SIZE);
4090  return x;
4091 }
4092 
4093 
4094 
4095 } // namespace cyclus
unsigned int val[CYCLUS_SHA1_NINT]
const Vals & vals()
Returns a vector of all field-value pairs that have been added to this datum.
Definition: datum.cc:44
std::string title()
Returns the datum&#39;s title as specified during the datum&#39;s creation.
Definition: datum.cc:40
const Shapes & shapes()
Returns a vector of all shapes (pointers to vectors of ints) that have been added to this datum...
Definition: datum.cc:48
virtual void Notify(DatumList data)
Used to pass a list of new/collected Datum objects.
Definition: hdf5_back.cc:77
Meta data and results of a query.
const void * castsmallvoid() const
Definition: any.hpp:319
std::vector< T > cast() const
Casts the value of this digest to a vector of the templated type.
For values that are too big, too small, etc.
Definition: error.h:41
std::string name(int nuc)
Definition: pyne.cc:2940
Digest digest()
std::vector< Shape > Shapes
Definition: datum.h:22
DbTypes
This is the master list of all supported database types.
Definition: query_backend.h:26
#define CYCLUS_SHA1_NINT
Definition: query_backend.h:17
virtual void Flush()
Flushes all buffered data in the backend to its final format/location.
Definition: hdf5_back.h:91
Used to specify and send a collection of key-value pairs to the Recorder for recording.
Definition: datum.h:15
The digest type for SHA1s.
std::vector< boost::spirit::hold_any > QueryRow
virtual std::string Name()
Used to uniquely identify a backend - particularly if there are more than one in a simulation...
Definition: hdf5_back.cc:1561
for failed reading/writing to files, network connections, etc..
Definition: error.h:59
std::vector< std::string > fields
names of each field returned by a query
std::vector< Entry > Vals
Definition: datum.h:20
std::vector< int > Shape
Definition: datum.h:21
std::vector< DbTypes > types
types of each field returned by a query.
std::string field
table column name
virtual void Close()
Closes and flushes the backend.
Definition: hdf5_back.cc:41
T const & cast() const
Definition: any.hpp:309
Hdf5Back(std::string path)
Creates a new backend writing data to the specified file.
Definition: hdf5_back.cc:11
virtual std::map< std::string, DbTypes > ColumnTypes(std::string table)
Return a map of column names of the specified table to the associated database type.
Definition: hdf5_back.cc:2634
taken directly from OsiSolverInterface.cpp on 2/17/14 from https://projects.coin-or.org/Osi/browser/trunk.
Definition: agent.cc:14
A type to represent variable-length array of bytes for dumping to a cyclus output database...
Definition: blob.h:9
virtual ~Hdf5Back()
cleans up resources and closes the file.
Definition: hdf5_back.cc:72
std::vector< QueryRow > rows
ordered results of a query
Represents a condition used to filter rows returned by a query.
std::vector< Datum * > DatumList
Definition: rec_backend.h:12
virtual std::set< std::string > Tables()
Return a set of all table names currently in the database.
Definition: hdf5_back.cc:2661
virtual QueryResult Query(std::string table, std::vector< Cond > *conds)
Return a set of rows from the specificed table that match all given conditions.
Definition: hdf5_back.cc:163
int offset(int dz, int da, int ds=0)
A helper function to compute nuclide id offsets from z-, a-, and s- deltas.
Definition: pyne.h:1455
#define CYCLUS_SHA1_SIZE
Definition: query_backend.h:16
void Clear()
Clears the current hash value to its default state.
void Update(const std::string &s)
Updates the hash value in-place.
#define CYCLUS_UUID_SIZE
Definition: query_backend.h:15