NetCDF  4.9.0
nc4hdf.c
Go to the documentation of this file.
1 /* Copyright 2018, University Corporation for Atmospheric
2  * Research. See the COPYRIGHT file for copying and redistribution
3  * conditions. */
17 #include "config.h"
18 #include "netcdf.h"
19 #include "nc4internal.h"
20 #include "ncdispatch.h"
21 #include "hdf5internal.h"
22 #include "hdf5err.h" /* For BAIL2 */
23 #include "hdf5debug.h"
24 #include <math.h>
25 
26 #ifdef HAVE_INTTYPES_H
27 #define __STDC_FORMAT_MACROS
28 #include <inttypes.h>
29 #endif
30 
31 #define NC_HDF5_MAX_NAME 1024
41 static int
42 flag_atts_dirty(NCindex *attlist) {
43 
44  NC_ATT_INFO_T *att = NULL;
45  int i;
46 
47  if(attlist == NULL) {
48  return NC_NOERR;
49  }
50 
51  for(i=0;i<ncindexsize(attlist);i++) {
52  att = (NC_ATT_INFO_T*)ncindexith(attlist,i);
53  if(att == NULL) continue;
54  att->dirty = NC_TRUE;
55  }
56 
57  return NC_NOERR;
58 }
59 
76 int
77 rec_reattach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid)
78 {
79  NC_VAR_INFO_T *var;
80  NC_GRP_INFO_T *child_grp;
81  int d, i;
82  int retval;
83 
84  assert(grp && grp->hdr.name && dimid >= 0 && dimscaleid >= 0);
85  LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
86 
87  /* If there are any child groups, attach dimscale there, if needed. */
88  for (i = 0; i < ncindexsize(grp->children); i++)
89  {
90  child_grp = (NC_GRP_INFO_T*)ncindexith(grp->children, i);
91  assert(child_grp);
92  if ((retval = rec_reattach_scales(child_grp, dimid, dimscaleid)))
93  return retval;
94  }
95 
96  /* Find any vars that use this dimension id. */
97  for (i = 0; i < ncindexsize(grp->vars); i++)
98  {
99  NC_HDF5_VAR_INFO_T *hdf5_var;
100 
101  var = (NC_VAR_INFO_T*)ncindexith(grp->vars,i);
102  assert(var && var->format_var_info);
103 
104  hdf5_var = (NC_HDF5_VAR_INFO_T*)var->format_var_info;
105  assert(hdf5_var != NULL);
106  for (d = 0; d < var->ndims; d++)
107  {
108  if (var->dimids[d] == dimid && !hdf5_var->dimscale)
109  {
110  LOG((2, "%s: attaching scale for dimid %d to var %s",
111  __func__, var->dimids[d], var->hdr.name));
112  if (var->created)
113  {
114  if (H5DSattach_scale(hdf5_var->hdf_datasetid,
115  dimscaleid, d) < 0)
116  return NC_EDIMSCALE;
117  hdf5_var->dimscale_attached[d] = NC_TRUE;
118  }
119  }
120  }
121  }
122  return NC_NOERR;
123 }
124 
141 int
142 rec_detach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid)
143 {
144  NC_VAR_INFO_T *var;
145  NC_GRP_INFO_T *child_grp;
146  int d, i;
147  int retval;
148 
149  assert(grp && grp->hdr.name && dimid >= 0 && dimscaleid >= 0);
150  LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
151 
152  /* If there are any child groups, detach dimscale there, if needed. */
153  for(i=0;i<ncindexsize(grp->children);i++) {
154  child_grp = (NC_GRP_INFO_T*)ncindexith(grp->children,i);
155  if(child_grp == NULL) continue;
156  if ((retval = rec_detach_scales(child_grp, dimid, dimscaleid)))
157  return retval;
158  }
159 
160  /* Find any vars that use this dimension id. */
161  for (i = 0; i < ncindexsize(grp->vars); i++)
162  {
163  NC_HDF5_VAR_INFO_T *hdf5_var;
164  var = (NC_VAR_INFO_T*)ncindexith(grp->vars, i);
165  assert(var && var->format_var_info);
166  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
167 
168  for (d = 0; d < var->ndims; d++)
169  {
170  if (var->dimids[d] == dimid && !hdf5_var->dimscale)
171  {
172  LOG((2, "%s: detaching scale for dimid %d to var %s",
173  __func__, var->dimids[d], var->hdr.name));
174  if (var->created)
175  {
176  if (hdf5_var->dimscale_attached && hdf5_var->dimscale_attached[d])
177  {
178  if (H5DSdetach_scale(hdf5_var->hdf_datasetid,
179  dimscaleid, d) < 0)
180  return NC_EDIMSCALE;
181  hdf5_var->dimscale_attached[d] = NC_FALSE;
182  }
183  }
184  }
185  }
186  }
187  return NC_NOERR;
188 }
189 
201 int
202 nc4_open_var_grp2(NC_GRP_INFO_T *grp, int varid, hid_t *dataset)
203 {
204  NC_VAR_INFO_T *var;
205  NC_HDF5_VAR_INFO_T *hdf5_var;
206 
207  assert(grp && grp->format_grp_info && dataset);
208 
209  /* Find the requested varid. */
210  if (!(var = (NC_VAR_INFO_T *)ncindexith(grp->vars, varid)))
211  return NC_ENOTVAR;
212  assert(var && var->hdr.id == varid && var->format_var_info);
213  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
214 
215  /* Open this dataset if necessary. */
216  if (!hdf5_var->hdf_datasetid)
217  {
218  NC_HDF5_GRP_INFO_T *hdf5_grp;
219  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
220 
221  if ((hdf5_var->hdf_datasetid = H5Dopen2(hdf5_grp->hdf_grpid,
222  var->hdr.name, H5P_DEFAULT)) < 0)
223  return NC_ENOTVAR;
224  }
225 
226  *dataset = hdf5_var->hdf_datasetid;
227 
228  return NC_NOERR;
229 }
230 
248 int
249 nc4_get_hdf_typeid(NC_FILE_INFO_T *h5, nc_type xtype,
250  hid_t *hdf_typeid, int endianness)
251 {
252  NC_TYPE_INFO_T *type;
253  hid_t typeid = 0;
254  int retval = NC_NOERR;
255 
256  assert(hdf_typeid && h5);
257 
258  *hdf_typeid = -1;
259 
260  /* Determine an appropriate HDF5 datatype */
261  if (xtype == NC_NAT)
262  return NC_EBADTYPE;
263  else if (xtype == NC_CHAR || xtype == NC_STRING)
264  {
265  /* NC_CHAR & NC_STRING types create a new HDF5 datatype */
266  if (xtype == NC_CHAR)
267  {
268  if ((typeid = H5Tcopy(H5T_C_S1)) < 0)
269  return NC_EHDFERR;
270  if (H5Tset_strpad(typeid, H5T_STR_NULLTERM) < 0)
271  BAIL(NC_EVARMETA);
272  if(H5Tset_cset(typeid, H5T_CSET_ASCII) < 0)
273  BAIL(NC_EVARMETA);
274 
275  /* Take ownership of the newly created HDF5 datatype */
276  *hdf_typeid = typeid;
277  typeid = 0;
278  }
279  else
280  {
281  if ((typeid = H5Tcopy(H5T_C_S1)) < 0)
282  return NC_EHDFERR;
283  if (H5Tset_size(typeid, H5T_VARIABLE) < 0)
284  BAIL(NC_EVARMETA);
285  if(H5Tset_cset(typeid, H5T_CSET_UTF8) < 0)
286  BAIL(NC_EVARMETA);
287 
288  /* Take ownership of the newly created HDF5 datatype */
289  *hdf_typeid = typeid;
290  typeid = 0;
291  }
292  }
293  else
294  {
295  /* All other types use an existing HDF5 datatype */
296  switch (xtype)
297  {
298  case NC_BYTE: /* signed 1 byte integer */
299  if (endianness == NC_ENDIAN_LITTLE)
300  typeid = H5T_STD_I8LE;
301  else if (endianness == NC_ENDIAN_BIG)
302  typeid = H5T_STD_I8BE;
303  else
304  typeid = H5T_NATIVE_SCHAR;
305  break;
306 
307  case NC_SHORT: /* signed 2 byte integer */
308  if (endianness == NC_ENDIAN_LITTLE)
309  typeid = H5T_STD_I16LE;
310  else if (endianness == NC_ENDIAN_BIG)
311  typeid = H5T_STD_I16BE;
312  else
313  typeid = H5T_NATIVE_SHORT;
314  break;
315 
316  case NC_INT:
317  if (endianness == NC_ENDIAN_LITTLE)
318  typeid = H5T_STD_I32LE;
319  else if (endianness == NC_ENDIAN_BIG)
320  typeid = H5T_STD_I32BE;
321  else
322  typeid = H5T_NATIVE_INT;
323  break;
324 
325  case NC_UBYTE:
326  if (endianness == NC_ENDIAN_LITTLE)
327  typeid = H5T_STD_U8LE;
328  else if (endianness == NC_ENDIAN_BIG)
329  typeid = H5T_STD_U8BE;
330  else
331  typeid = H5T_NATIVE_UCHAR;
332  break;
333 
334  case NC_USHORT:
335  if (endianness == NC_ENDIAN_LITTLE)
336  typeid = H5T_STD_U16LE;
337  else if (endianness == NC_ENDIAN_BIG)
338  typeid = H5T_STD_U16BE;
339  else
340  typeid = H5T_NATIVE_USHORT;
341  break;
342 
343  case NC_UINT:
344  if (endianness == NC_ENDIAN_LITTLE)
345  typeid = H5T_STD_U32LE;
346  else if (endianness == NC_ENDIAN_BIG)
347  typeid = H5T_STD_U32BE;
348  else
349  typeid = H5T_NATIVE_UINT;
350  break;
351 
352  case NC_INT64:
353  if (endianness == NC_ENDIAN_LITTLE)
354  typeid = H5T_STD_I64LE;
355  else if (endianness == NC_ENDIAN_BIG)
356  typeid = H5T_STD_I64BE;
357  else
358  typeid = H5T_NATIVE_LLONG;
359  break;
360 
361  case NC_UINT64:
362  if (endianness == NC_ENDIAN_LITTLE)
363  typeid = H5T_STD_U64LE;
364  else if (endianness == NC_ENDIAN_BIG)
365  typeid = H5T_STD_U64BE;
366  else
367  typeid = H5T_NATIVE_ULLONG;
368  break;
369 
370  case NC_FLOAT:
371  if (endianness == NC_ENDIAN_LITTLE)
372  typeid = H5T_IEEE_F32LE;
373  else if (endianness == NC_ENDIAN_BIG)
374  typeid = H5T_IEEE_F32BE;
375  else
376  typeid = H5T_NATIVE_FLOAT;
377  break;
378 
379  case NC_DOUBLE:
380  if (endianness == NC_ENDIAN_LITTLE)
381  typeid = H5T_IEEE_F64LE;
382  else if (endianness == NC_ENDIAN_BIG)
383  typeid = H5T_IEEE_F64BE;
384  else
385  typeid = H5T_NATIVE_DOUBLE;
386  break;
387 
388  default:
389  /* Maybe this is a user defined type? */
390  if (nc4_find_type(h5, xtype, &type))
391  return NC_EBADTYPE;
392  if (!type)
393  return NC_EBADTYPE;
394  typeid = ((NC_HDF5_TYPE_INFO_T *)type->format_type_info)->hdf_typeid;
395  break;
396  }
397  assert(typeid);
398 
399  /* Copy the HDF5 datatype, so the function operates uniformly */
400  if ((*hdf_typeid = H5Tcopy(typeid)) < 0)
401  return NC_EHDFERR;
402  typeid = 0;
403  }
404  assert(*hdf_typeid != -1);
405 
406 exit:
407  if (typeid > 0 && H5Tclose(typeid) < 0)
408  BAIL2(NC_EHDFERR);
409  return retval;
410 }
411 
426 static int
427 put_att_grpa(NC_GRP_INFO_T *grp, int varid, NC_ATT_INFO_T *att)
428 {
429  NC_HDF5_GRP_INFO_T *hdf5_grp;
430  hid_t datasetid = 0, locid;
431  hid_t attid = 0, spaceid = 0, file_typeid = 0;
432  hid_t existing_att_typeid = 0, existing_attid = 0, existing_spaceid = 0;
433  hsize_t dims[1]; /* netcdf attributes always 1-D. */
434  htri_t attr_exists;
435  void *data;
436  int phoney_data = 99;
437  int retval = NC_NOERR;
438 
439  assert(att->hdr.name && grp && grp->format_grp_info);
440  LOG((3, "%s: varid %d att->hdr.id %d att->hdr.name %s att->nc_typeid %d "
441  "att->len %d", __func__, varid, att->hdr.id, att->hdr.name,
442  att->nc_typeid, att->len));
443 
444  /* Get HDF5-specific group info. */
445  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
446 
447  /* If the file is read-only, return an error. */
448  if (grp->nc4_info->no_write)
449  BAIL(NC_EPERM);
450 
451  /* Get the hid to attach the attribute to, or read it from. */
452  if (varid == NC_GLOBAL)
453  locid = hdf5_grp->hdf_grpid;
454  else
455  {
456  if ((retval = nc4_open_var_grp2(grp, varid, &datasetid)))
457  BAIL(retval);
458  locid = datasetid;
459  }
460 
461  /* Get the length ready, and find the HDF type we'll be
462  * writing. */
463  dims[0] = att->len;
464  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, att->nc_typeid,
465  &file_typeid, 0)))
466  BAIL(retval);
467 
468  /* Even if the length is zero, HDF5 won't let me write with a
469  * NULL pointer. So if the length of the att is zero, point to
470  * some phoney data (which won't be written anyway.)*/
471  if (!dims[0])
472  data = &phoney_data;
473 #ifdef SEPDATA
474  else if (att->vldata)
475  data = att->vldata;
476  else if (att->stdata)
477  data = att->stdata;
478 #endif
479  else
480  data = att->data;
481 
482  /* NC_CHAR types require some extra work. The space ID is set to
483  * scalar, and the type is told how long the string is. If it's
484  * really zero length, set the size to 1. (The fact that it's
485  * really zero will be marked by the NULL dataspace, but HDF5
486  * doesn't allow me to set the size of the type to zero.)*/
487  if (att->nc_typeid == NC_CHAR)
488  {
489  size_t string_size = dims[0];
490  if (!string_size)
491  {
492  string_size = 1;
493  if ((spaceid = H5Screate(H5S_NULL)) < 0)
494  BAIL(NC_EATTMETA);
495  }
496  else
497  {
498  if ((spaceid = H5Screate(H5S_SCALAR)) < 0)
499  BAIL(NC_EATTMETA);
500  }
501  if (H5Tset_size(file_typeid, string_size) < 0)
502  BAIL(NC_EATTMETA);
503  if (H5Tset_strpad(file_typeid, H5T_STR_NULLTERM) < 0)
504  BAIL(NC_EATTMETA);
505  }
506  else
507  {
508  if (!att->len)
509  {
510  if ((spaceid = H5Screate(H5S_NULL)) < 0)
511  BAIL(NC_EATTMETA);
512  }
513  else
514  {
515  if ((spaceid = H5Screate_simple(1, dims, NULL)) < 0)
516  BAIL(NC_EATTMETA);
517  }
518  }
519 
520  /* Does the att exists already? */
521  if ((attr_exists = H5Aexists(locid, att->hdr.name)) < 0)
522  BAIL(NC_EHDFERR);
523  if (attr_exists)
524  {
525  hssize_t npoints;
526 
527  /* Open the attribute. */
528  if ((existing_attid = H5Aopen(locid, att->hdr.name, H5P_DEFAULT)) < 0)
529  BAIL(NC_EATTMETA);
530 
531  /* Find the type of the existing attribute. */
532  if ((existing_att_typeid = H5Aget_type(existing_attid)) < 0)
533  BAIL(NC_EATTMETA);
534 
535  /* How big is the attribute? */
536  if ((existing_spaceid = H5Aget_space(existing_attid)) < 0)
537  BAIL(NC_EATTMETA);
538  if ((npoints = H5Sget_simple_extent_npoints(existing_spaceid)) < 0)
539  BAIL(NC_EATTMETA);
540 
541  /* For text attributes the size is specified in the datatype
542  and it is enough to compare types using H5Tequal(). */
543  if (!H5Tequal(file_typeid, existing_att_typeid) ||
544  (att->nc_typeid != NC_CHAR && npoints != att->len))
545  {
546  /* The attribute exists but we cannot re-use it. */
547 
548  /* Delete the attribute. */
549  if (H5Adelete(locid, att->hdr.name) < 0)
550  BAIL(NC_EHDFERR);
551 
552  /* Re-create the attribute with the type and length
553  reflecting the new value (or values). */
554  if ((attid = H5Acreate1(locid, att->hdr.name, file_typeid, spaceid,
555  H5P_DEFAULT)) < 0)
556  BAIL(NC_EATTMETA);
557 
558  /* Write the values, (even if length is zero). */
559  if (H5Awrite(attid, file_typeid, data) < 0)
560  BAIL(NC_EATTMETA);
561  }
562  else
563  {
564  /* The attribute exists and we can re-use it. */
565 
566  /* Write the values, re-using the existing attribute. */
567  if (H5Awrite(existing_attid, file_typeid, data) < 0)
568  BAIL(NC_EATTMETA);
569  }
570  }
571  else
572  {
573  /* The attribute does not exist yet. */
574 
575  /* Create the attribute. */
576  if ((attid = H5Acreate1(locid, att->hdr.name, file_typeid, spaceid,
577  H5P_DEFAULT)) < 0)
578  BAIL(NC_EATTMETA);
579 
580  /* Write the values, (even if length is zero). */
581  if (H5Awrite(attid, file_typeid, data) < 0)
582  BAIL(NC_EATTMETA);
583  }
584 
585 exit:
586  if (file_typeid && H5Tclose(file_typeid))
587  BAIL2(NC_EHDFERR);
588  if (attid > 0 && H5Aclose(attid) < 0)
589  BAIL2(NC_EHDFERR);
590  if (existing_att_typeid && H5Tclose(existing_att_typeid))
591  BAIL2(NC_EHDFERR);
592  if (existing_attid > 0 && H5Aclose(existing_attid) < 0)
593  BAIL2(NC_EHDFERR);
594  if (spaceid > 0 && H5Sclose(spaceid) < 0)
595  BAIL2(NC_EHDFERR);
596  if (existing_spaceid > 0 && H5Sclose(existing_spaceid) < 0)
597  BAIL2(NC_EHDFERR);
598  return retval;
599 }
600 
612 static int
613 write_attlist(NCindex *attlist, int varid, NC_GRP_INFO_T *grp)
614 {
615  NC_ATT_INFO_T *att;
616  int retval;
617  int i;
618 
619  for(i = 0; i < ncindexsize(attlist); i++)
620  {
621  att = (NC_ATT_INFO_T *)ncindexith(attlist, i);
622  assert(att);
623  if (att->dirty)
624  {
625  LOG((4, "%s: writing att %s to varid %d", __func__, att->hdr.name, varid));
626  if ((retval = put_att_grpa(grp, varid, att)))
627  return retval;
628  att->dirty = NC_FALSE;
629  att->created = NC_TRUE;
630  }
631  }
632  return NC_NOERR;
633 }
634 
648 static int
649 write_coord_dimids(NC_VAR_INFO_T *var)
650 {
651  NC_HDF5_VAR_INFO_T *hdf5_var;
652  hsize_t coords_len[1];
653  hid_t c_spaceid = -1, c_attid = -1;
654  int retval = NC_NOERR;
655 
656  assert(var && var->format_var_info);
657 
658  /* Get HDF5-specific var info. */
659  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
660 
661  /* Set up space for attribute. */
662  coords_len[0] = var->ndims;
663  if ((c_spaceid = H5Screate_simple(1, coords_len, coords_len)) < 0)
664  BAIL(NC_EHDFERR);
665 
666  /* Create the attribute. */
667  if ((c_attid = H5Acreate1(hdf5_var->hdf_datasetid, COORDINATES,
668  H5T_NATIVE_INT, c_spaceid, H5P_DEFAULT)) < 0)
669  BAIL(NC_EHDFERR);
670 
671  /* Write our attribute. */
672  if (H5Awrite(c_attid, H5T_NATIVE_INT, var->dimids) < 0)
673  BAIL(NC_EHDFERR);
674 
675 exit:
676  if (c_spaceid >= 0 && H5Sclose(c_spaceid) < 0)
677  BAIL2(NC_EHDFERR);
678  if (c_attid >= 0 && H5Aclose(c_attid) < 0)
679  BAIL2(NC_EHDFERR);
680  return retval;
681 }
682 
693 static int
694 write_netcdf4_dimid(hid_t datasetid, int dimid)
695 {
696  hid_t dimid_spaceid = -1, dimid_attid = -1;
697  htri_t attr_exists;
698  int retval = NC_NOERR;
699 
700  /* Create the space. */
701  if ((dimid_spaceid = H5Screate(H5S_SCALAR)) < 0)
702  BAIL(NC_EHDFERR);
703 
704  /* Does the attribute already exist? If so, don't try to create it. */
705  if ((attr_exists = H5Aexists(datasetid, NC_DIMID_ATT_NAME)) < 0)
706  BAIL(NC_EHDFERR);
707  if (attr_exists)
708  dimid_attid = H5Aopen_by_name(datasetid, ".", NC_DIMID_ATT_NAME,
709  H5P_DEFAULT, H5P_DEFAULT);
710  else
711  /* Create the attribute if needed. */
712  dimid_attid = H5Acreate1(datasetid, NC_DIMID_ATT_NAME,
713  H5T_NATIVE_INT, dimid_spaceid, H5P_DEFAULT);
714  if (dimid_attid < 0)
715  BAIL(NC_EHDFERR);
716 
717 
718  /* Write it. */
719  LOG((4, "%s: writing secret dimid %d", __func__, dimid));
720  if (H5Awrite(dimid_attid, H5T_NATIVE_INT, &dimid) < 0)
721  BAIL(NC_EHDFERR);
722 
723 exit:
724  /* Close stuff*/
725  if (dimid_spaceid >= 0 && H5Sclose(dimid_spaceid) < 0)
726  BAIL2(NC_EHDFERR);
727  if (dimid_attid >= 0 && H5Aclose(dimid_attid) < 0)
728  BAIL2(NC_EHDFERR);
729 
730  return retval;
731 }
732 
747 static int
748 var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid)
749 {
750  NC_HDF5_GRP_INFO_T *hdf5_grp;
751  NC_HDF5_VAR_INFO_T *hdf5_var;
752  hid_t plistid = 0, access_plistid = 0, typeid = 0, spaceid = 0;
753  hsize_t chunksize[H5S_MAX_RANK], dimsize[H5S_MAX_RANK], maxdimsize[H5S_MAX_RANK];
754  int d;
755  void *fillp = NULL;
756  NC_DIM_INFO_T *dim = NULL;
757  char *name_to_use;
758  int retval;
759  unsigned int* params = NULL;
760 
761  assert(grp && grp->format_grp_info && var && var->format_var_info);
762 
763  LOG((3, "%s:: name %s", __func__, var->hdr.name));
764 
765  /* Get HDF5-specific group and var info. */
766  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
767  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
768 
769  /* Scalar or not, we need a creation property list. */
770  if ((plistid = H5Pcreate(H5P_DATASET_CREATE)) < 0)
771  BAIL(NC_EHDFERR);
772  if ((access_plistid = H5Pcreate(H5P_DATASET_ACCESS)) < 0)
773  BAIL(NC_EHDFERR);
774 
775  /* Turn off object tracking times in HDF5. */
776  if (H5Pset_obj_track_times(plistid, 0) < 0)
777  BAIL(NC_EHDFERR);
778 
779  /* Find the HDF5 type of the dataset. */
780  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, var->type_info->hdr.id, &typeid,
781  var->type_info->endianness)))
782  BAIL(retval);
783 
784  /* Figure out what fill value to set, if any. */
785  if (var->no_fill)
786  {
787  /* Required to truly turn HDF5 fill values off */
788  if (H5Pset_fill_time(plistid, H5D_FILL_TIME_NEVER) < 0)
789  BAIL(NC_EHDFERR);
790  }
791  else
792  {
793  if ((retval = nc4_get_fill_value(grp->nc4_info, var, &fillp)))
794  BAIL(retval);
795 
796  /* If there is a fill value, set it. */
797  if (fillp)
798  {
799  if (var->type_info->nc_type_class == NC_STRING)
800  {
801  if (H5Pset_fill_value(plistid, typeid, fillp) < 0)
802  BAIL(NC_EHDFERR);
803  }
804  else
805  {
806  /* The fill value set in HDF5 must always be presented as
807  * a native type, even if the endianness for this dataset
808  * is non-native. HDF5 will translate the fill value to
809  * the target endiannesss. */
810  hid_t fill_typeid = 0;
811 
812  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, var->type_info->hdr.id, &fill_typeid,
814  BAIL(retval);
815  if (H5Pset_fill_value(plistid, fill_typeid, fillp) < 0)
816  {
817  if (H5Tclose(fill_typeid) < 0)
818  BAIL(NC_EHDFERR);
819  BAIL(NC_EHDFERR);
820  }
821  if (H5Tclose(fill_typeid) < 0)
822  BAIL(NC_EHDFERR);
823  }
824  }
825  }
826 
827  /* If the user wants to compress the data, using either zlib
828  * (a.k.a deflate) or szip, or another filter, set that up now.
829  * Szip and zip can be turned on
830  * either directly with nc_def_var_szip/deflate(), or using
831  * nc_def_var_filter(). If the user
832  * has specified a filter, it will be applied here. */
833  if(var->filters != NULL) {
834  int j;
835  NClist* filters = (NClist*)var->filters;
836  for(j=0;j<nclistlength(filters);j++) {
837  struct NC_HDF5_Filter* fi = (struct NC_HDF5_Filter*)nclistget(filters,j);
838  if(fi->filterid == H5Z_FILTER_FLETCHER32) {
839  if(H5Pset_fletcher32(plistid) < 0)
840  BAIL(NC_EHDFERR);
841  } else if(fi->filterid == H5Z_FILTER_SHUFFLE) {
842  if(H5Pset_shuffle(plistid) < 0)
843  BAIL(NC_EHDFERR);
844  } else if(fi->filterid == H5Z_FILTER_DEFLATE) {/* Handle zip case here */
845  unsigned level;
846  if(fi->nparams != 1)
847  BAIL(NC_EFILTER);
848  level = (int)fi->params[0];
849  if(H5Pset_deflate(plistid, level) < 0)
850  BAIL(NC_EFILTER);
851  } else if(fi->filterid == H5Z_FILTER_SZIP) {/* Handle szip case here */
852  int options_mask;
853  int bits_per_pixel;
854  if(fi->nparams != 2)
855  BAIL(NC_EFILTER);
856  options_mask = (int)fi->params[0];
857  bits_per_pixel = (int)fi->params[1];
858  if(H5Pset_szip(plistid, options_mask, bits_per_pixel) < 0)
859  BAIL(NC_EFILTER);
860  } else {
861  herr_t code = H5Pset_filter(plistid, fi->filterid,
862 #if 1
863  H5Z_FLAG_MANDATORY,
864 #else
865  H5Z_FLAG_OPTIONAL,
866 #endif
867  fi->nparams, fi->params);
868  if(code < 0)
869  BAIL(NC_EFILTER);
870  }
871  }
872  }
873 
874  /* If ndims non-zero, get info for all dimensions. We look up the
875  dimids and get the len of each dimension. We need this to create
876  the space for the dataset. In netCDF a dimension length of zero
877  means an unlimited dimension. */
878  if (var->ndims)
879  {
880  int unlimdim = 0;
881 
882  /* Check to see if any unlimited dimensions are used in this var. */
883  for (d = 0; d < var->ndims; d++) {
884  dim = var->dim[d];
885  assert(dim && dim->hdr.id == var->dimids[d]);
886  if (dim->unlimited)
887  unlimdim++;
888  }
889 
890  /* If there are no unlimited dims, and no filters, and the user
891  * has not specified chunksizes, use contiguous variable for
892  * better performance. */
893  if (nclistlength((NClist*)var->filters) == 0 &&
894  (var->chunksizes == NULL || !var->chunksizes[0]) && !unlimdim)
895  var->storage = NC_CONTIGUOUS;
896 
897  /* Gather current & maximum dimension sizes, along with chunk
898  * sizes. */
899  for (d = 0; d < var->ndims; d++)
900  {
901  dim = var->dim[d];
902  assert(dim && dim->hdr.id == var->dimids[d]);
903  dimsize[d] = dim->unlimited ? NC_HDF5_UNLIMITED_DIMSIZE : dim->len;
904  maxdimsize[d] = dim->unlimited ? H5S_UNLIMITED : (hsize_t)dim->len;
905  if (var->storage == NC_CHUNKED)
906  {
907  if (var->chunksizes[d])
908  chunksize[d] = var->chunksizes[d];
909  else
910  {
911  size_t type_size;
912  if (var->type_info->nc_type_class == NC_STRING)
913  type_size = sizeof(char *);
914  else
915  type_size = var->type_info->size;
916 
917  /* Unlimited dim always gets chunksize of 1. */
918  if (dim->unlimited)
919  chunksize[d] = 1;
920  else
921  chunksize[d] = pow((double)DEFAULT_CHUNK_SIZE/type_size,
922  1/(double)(var->ndims - unlimdim));
923 
924  /* If the chunksize is greater than the dim
925  * length, make it the dim length. */
926  if (!dim->unlimited && chunksize[d] > dim->len)
927  chunksize[d] = dim->len;
928 
929  /* Remember the computed chunksize */
930  var->chunksizes[d] = chunksize[d];
931  }
932  }
933  }
934 
935  /* Create the dataspace. */
936  if ((spaceid = H5Screate_simple(var->ndims, dimsize, maxdimsize)) < 0)
937  BAIL(NC_EHDFERR);
938  }
939  else
940  {
941  if ((spaceid = H5Screate(H5S_SCALAR)) < 0)
942  BAIL(NC_EHDFERR);
943  }
944 
945  /* Set the var storage to contiguous, compact, or chunked. Don't
946  * try to set chunking for scalar vars, they will default to
947  * contiguous if not set to compact. */
948  if (var->storage == NC_CONTIGUOUS)
949  {
950  if (H5Pset_layout(plistid, H5D_CONTIGUOUS) < 0)
951  BAIL(NC_EHDFERR);
952  }
953  else if (var->storage == NC_COMPACT)
954  {
955  if (H5Pset_layout(plistid, H5D_COMPACT) < 0)
956  BAIL(NC_EHDFERR);
957  }
958  else if (var->ndims)
959  {
960  if (H5Pset_chunk(plistid, var->ndims, chunksize) < 0)
961  BAIL(NC_EHDFERR);
962  }
963 
964  /* Turn on creation order tracking. */
965  if (!grp->nc4_info->no_attr_create_order) {
966  if (H5Pset_attr_creation_order(plistid, H5P_CRT_ORDER_TRACKED|
967  H5P_CRT_ORDER_INDEXED) < 0)
968  BAIL(NC_EHDFERR);
969  }
970 
971  /* Set per-var chunk cache, for chunked datasets. */
972  if (var->storage == NC_CHUNKED && var->chunkcache.size)
973  if (H5Pset_chunk_cache(access_plistid, var->chunkcache.nelems,
974  var->chunkcache.size, var->chunkcache.preemption) < 0)
975  BAIL(NC_EHDFERR);
976 
977  /* At long last, create the dataset. */
978  name_to_use = var->alt_name ? var->alt_name : var->hdr.name;
979  LOG((4, "%s: about to H5Dcreate2 dataset %s of type 0x%x", __func__,
980  name_to_use, typeid));
981  if ((hdf5_var->hdf_datasetid = H5Dcreate2(hdf5_grp->hdf_grpid, name_to_use, typeid,
982  spaceid, H5P_DEFAULT, plistid, access_plistid)) < 0)
983  BAIL(NC_EHDFERR);
984  var->created = NC_TRUE;
985  var->is_new_var = NC_FALSE;
986 
987  /* Always write the hidden coordinates attribute, which lists the
988  * dimids of this var. When present, this speeds opens. When not
989  * present, dimscale matching is used. */
990  if (var->ndims)
991  if ((retval = write_coord_dimids(var)))
992  BAIL(retval);
993 
994  /* If this is a dimscale, mark it as such in the HDF5 file. Also
995  * find the dimension info and store the dataset id of the dimscale
996  * dataset. */
997  if (hdf5_var->dimscale)
998  {
999  if (H5DSset_scale(hdf5_var->hdf_datasetid, var->hdr.name) < 0)
1000  BAIL(NC_EHDFERR);
1001 
1002  /* If this is a multidimensional coordinate variable, write a
1003  * coordinates attribute. */
1004  /* if (var->ndims > 1) */
1005  /* if ((retval = write_coord_dimids(var))) */
1006  /* BAIL(retval); */
1007 
1008  /* If desired, write the netCDF dimid. */
1009  if (write_dimid)
1010  if ((retval = write_netcdf4_dimid(hdf5_var->hdf_datasetid, var->dimids[0])))
1011  BAIL(retval);
1012  }
1013 
1014  /* If quantization is in use, write an attribute indicating it, a
1015  * single integer which is the number of significant digits
1016  * (NSD, for BitGroom and Granular BitRound) or number of significant bits
1017  * (NSB, for BitRound). */
1018  if (var->quantize_mode == NC_QUANTIZE_BITGROOM)
1019  if ((retval = nc4_put_att(var->container, var->hdr.id, NC_QUANTIZE_BITGROOM_ATT_NAME, NC_INT, 1,
1020  &var->nsd, NC_INT, 0)))
1021  BAIL(retval);
1022 
1023  if (var->quantize_mode == NC_QUANTIZE_GRANULARBR)
1024  if ((retval = nc4_put_att(var->container, var->hdr.id, NC_QUANTIZE_GRANULARBR_ATT_NAME, NC_INT, 1,
1025  &var->nsd, NC_INT, 0)))
1026  BAIL(retval);
1027 
1028  if (var->quantize_mode == NC_QUANTIZE_BITROUND)
1029  if ((retval = nc4_put_att(var->container, var->hdr.id, NC_QUANTIZE_BITROUND_ATT_NAME, NC_INT, 1,
1030  &var->nsd, NC_INT, 0)))
1031  BAIL(retval);
1032 
1033  /* Write attributes for this var. */
1034  if ((retval = write_attlist(var->att, var->hdr.id, grp)))
1035  BAIL(retval);
1036 
1037  /* The file is now up-to-date with all settings for this var. */
1038  var->attr_dirty = NC_FALSE;
1039 
1040 exit:
1041  nullfree(params);
1042  if (typeid > 0 && H5Tclose(typeid) < 0)
1043  BAIL2(NC_EHDFERR);
1044  if (plistid > 0 && H5Pclose(plistid) < 0)
1045  BAIL2(NC_EHDFERR);
1046  if (access_plistid > 0 && H5Pclose(access_plistid) < 0)
1047  BAIL2(NC_EHDFERR);
1048  if (spaceid > 0 && H5Sclose(spaceid) < 0)
1049  BAIL2(NC_EHDFERR);
1050  if (fillp)
1051  {
1052  if (var->type_info->nc_type_class == NC_VLEN)
1053  nc_free_vlen((nc_vlen_t *)fillp);
1054  else if (var->type_info->nc_type_class == NC_STRING && *(char **)fillp)
1055  free(*(char **)fillp);
1056  free(fillp);
1057  }
1058 
1059  return retval;
1060 }
1061 
1075 int
1076 nc4_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var)
1077 {
1078  size_t chunk_size_bytes = 1;
1079  int d;
1080  int retval;
1081 
1082  /* Nothing to be done for contiguous or compact data. */
1083  if (var->storage != NC_CHUNKED)
1084  return NC_NOERR;
1085 
1086 #ifdef USE_PARALLEL4
1087  /* Don't set cache for files using parallel I/O. */
1088  if (grp->nc4_info->parallel)
1089  return NC_NOERR;
1090 #endif
1091 
1092  /* How many bytes in the chunk? */
1093  for (d = 0; d < var->ndims; d++)
1094  chunk_size_bytes *= var->chunksizes[d];
1095  if (var->type_info->size)
1096  chunk_size_bytes *= var->type_info->size;
1097  else
1098  chunk_size_bytes *= sizeof(char *);
1099 
1100  /* If the chunk cache is too small, and the user has not changed
1101  * the default value of the chunk cache size, then increase the
1102  * size of the cache. */
1103  if (var->chunkcache.size == CHUNK_CACHE_SIZE)
1104  if (chunk_size_bytes > var->chunkcache.size)
1105  {
1106  var->chunkcache.size = chunk_size_bytes * DEFAULT_CHUNKS_IN_CACHE;
1107  if (var->chunkcache.size > MAX_DEFAULT_CACHE_SIZE)
1108  var->chunkcache.size = MAX_DEFAULT_CACHE_SIZE;
1109  if ((retval = nc4_reopen_dataset(grp, var)))
1110  return retval;
1111  }
1112 
1113  return NC_NOERR;
1114 }
1115 
1131 static int
1132 commit_type(NC_GRP_INFO_T *grp, NC_TYPE_INFO_T *type)
1133 {
1134  NC_HDF5_GRP_INFO_T *hdf5_grp;
1135  NC_HDF5_TYPE_INFO_T *hdf5_type;
1136  hid_t base_hdf_typeid;
1137  int retval;
1138 
1139  assert(grp && grp->format_grp_info && type && type->format_type_info);
1140 
1141  /* Get HDF5-specific group and type info. */
1142  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
1143  hdf5_type = (NC_HDF5_TYPE_INFO_T *)type->format_type_info;
1144 
1145  /* Did we already record this type? */
1146  if (type->committed)
1147  return NC_NOERR;
1148 
1149  /* Is this a compound type? */
1150  if (type->nc_type_class == NC_COMPOUND)
1151  {
1152  NC_FIELD_INFO_T *field;
1153  hid_t hdf_base_typeid, hdf_typeid;
1154  int i;
1155 
1156  if ((hdf5_type->hdf_typeid = H5Tcreate(H5T_COMPOUND, type->size)) < 0)
1157  return NC_EHDFERR;
1158  LOG((4, "creating compound type %s hdf_typeid 0x%x", type->hdr.name,
1159  hdf5_type->hdf_typeid));
1160 
1161  for(i=0;i<nclistlength(type->u.c.field);i++)
1162  {
1163  field = (NC_FIELD_INFO_T *)nclistget(type->u.c.field, i);
1164  assert(field);
1165  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, field->nc_typeid,
1166  &hdf_base_typeid, type->endianness)))
1167  return retval;
1168 
1169  /* If this is an array, create a special array type. */
1170  if (field->ndims)
1171  {
1172  int d;
1173  hsize_t dims[NC_MAX_VAR_DIMS];
1174 
1175  for (d = 0; d < field->ndims; d++)
1176  dims[d] = field->dim_size[d];
1177  if ((hdf_typeid = H5Tarray_create1(hdf_base_typeid, field->ndims,
1178  dims, NULL)) < 0)
1179  {
1180  if (H5Tclose(hdf_base_typeid) < 0)
1181  return NC_EHDFERR;
1182  return NC_EHDFERR;
1183  }
1184  if (H5Tclose(hdf_base_typeid) < 0)
1185  return NC_EHDFERR;
1186  }
1187  else
1188  hdf_typeid = hdf_base_typeid;
1189  LOG((4, "inserting field %s offset %d hdf_typeid 0x%x", field->hdr.name,
1190  field->offset, hdf_typeid));
1191  if (H5Tinsert(hdf5_type->hdf_typeid, field->hdr.name, field->offset,
1192  hdf_typeid) < 0)
1193  return NC_EHDFERR;
1194  if (H5Tclose(hdf_typeid) < 0)
1195  return NC_EHDFERR;
1196  }
1197  }
1198  else if (type->nc_type_class == NC_VLEN)
1199  {
1200  /* Find the HDF typeid of the base type of this vlen. */
1201  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, type->u.v.base_nc_typeid,
1202  &base_hdf_typeid, type->endianness)))
1203  return retval;
1204 
1205  /* Create a vlen type. */
1206  if ((hdf5_type->hdf_typeid = H5Tvlen_create(base_hdf_typeid)) < 0)
1207  return NC_EHDFERR;
1208  }
1209  else if (type->nc_type_class == NC_OPAQUE)
1210  {
1211  /* Create the opaque type. */
1212  if ((hdf5_type->hdf_typeid = H5Tcreate(H5T_OPAQUE, type->size)) < 0)
1213  return NC_EHDFERR;
1214  }
1215  else if (type->nc_type_class == NC_ENUM)
1216  {
1217  NC_ENUM_MEMBER_INFO_T *enum_m;
1218  int i;
1219 
1220  if (nclistlength(type->u.e.enum_member) == 0)
1221  return NC_EINVAL;
1222 
1223  /* Find the HDF typeid of the base type of this enum. */
1224  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, type->u.e.base_nc_typeid,
1225  &base_hdf_typeid, type->endianness)))
1226  return retval;
1227 
1228  /* Create an enum type. */
1229  if ((hdf5_type->hdf_typeid = H5Tenum_create(base_hdf_typeid)) < 0)
1230  return NC_EHDFERR;
1231 
1232  /* Add all the members to the HDF5 type. */
1233  for(i=0;i<nclistlength(type->u.e.enum_member);i++) {
1234  enum_m = (NC_ENUM_MEMBER_INFO_T*)nclistget(type->u.e.enum_member,i);
1235  if (H5Tenum_insert(hdf5_type->hdf_typeid, enum_m->name, enum_m->value) < 0)
1236  return NC_EHDFERR;
1237  }
1238  }
1239  else
1240  {
1241  LOG((0, "Unknown class: %d", type->nc_type_class));
1242  return NC_EBADTYPE;
1243  }
1244 
1245  /* Commit the type. */
1246  if (H5Tcommit1(hdf5_grp->hdf_grpid, type->hdr.name, hdf5_type->hdf_typeid) < 0)
1247  return NC_EHDFERR;
1248  type->committed = NC_TRUE;
1249  LOG((4, "just committed type %s, HDF typeid: 0x%x", type->hdr.name,
1250  hdf5_type->hdf_typeid));
1251 
1252  /* Later we will always use the native typeid. In this case, it is
1253  * a copy of the same type pointed to by hdf_typeid, but it's
1254  * easier to maintain a copy. */
1255  if ((hdf5_type->native_hdf_typeid = H5Tget_native_type(hdf5_type->hdf_typeid,
1256  H5T_DIR_DEFAULT)) < 0)
1257  return NC_EHDFERR;
1258 
1259  return NC_NOERR;
1260 }
1261 
1272 static int
1273 write_nc3_strict_att(hid_t hdf_grpid)
1274 {
1275  hid_t attid = 0, spaceid = 0;
1276  int one = 1;
1277  int retval = NC_NOERR;
1278  htri_t attr_exists;
1279 
1280  /* If the attribute already exists, call that a success and return
1281  * NC_NOERR. */
1282  if ((attr_exists = H5Aexists(hdf_grpid, NC3_STRICT_ATT_NAME)) < 0)
1283  return NC_EHDFERR;
1284  if (attr_exists)
1285  return NC_NOERR;
1286 
1287  /* Create the attribute to mark this as a file that needs to obey
1288  * strict netcdf-3 rules. */
1289  if ((spaceid = H5Screate(H5S_SCALAR)) < 0)
1290  BAIL(NC_EFILEMETA);
1291  if ((attid = H5Acreate1(hdf_grpid, NC3_STRICT_ATT_NAME,
1292  H5T_NATIVE_INT, spaceid, H5P_DEFAULT)) < 0)
1293  BAIL(NC_EFILEMETA);
1294  if (H5Awrite(attid, H5T_NATIVE_INT, &one) < 0)
1295  BAIL(NC_EFILEMETA);
1296 
1297 exit:
1298  if (spaceid > 0 && (H5Sclose(spaceid) < 0))
1299  BAIL2(NC_EFILEMETA);
1300  if (attid > 0 && (H5Aclose(attid) < 0))
1301  BAIL2(NC_EFILEMETA);
1302  return retval;
1303 }
1304 
1317 static int
1318 create_group(NC_GRP_INFO_T *grp)
1319 {
1320  NC_HDF5_GRP_INFO_T *hdf5_grp, *parent_hdf5_grp;
1321  hid_t gcpl_id = -1;
1322  int retval = NC_NOERR;;
1323 
1324  assert(grp && grp->format_grp_info && grp->parent &&
1325  grp->parent->format_grp_info);
1326 
1327  /* Get HDF5 specific group info for group and parent. */
1328  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
1329  parent_hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->parent->format_grp_info;
1330  assert(parent_hdf5_grp->hdf_grpid);
1331 
1332  /* Create group, with link_creation_order set in the group
1333  * creation property list. */
1334  if ((gcpl_id = H5Pcreate(H5P_GROUP_CREATE)) < 0)
1335  BAIL(NC_EHDFERR);
1336 
1337  /* Set track_times to be FALSE. */
1338  if (H5Pset_obj_track_times(gcpl_id, 0) < 0)
1339  BAIL(NC_EHDFERR);
1340 
1341  /* Tell HDF5 to keep track of objects in creation order. */
1342  if (H5Pset_link_creation_order(gcpl_id, H5P_CRT_ORDER_TRACKED|H5P_CRT_ORDER_INDEXED) < 0)
1343  BAIL(NC_EHDFERR);
1344 
1345  /* Tell HDF5 to keep track of attributes in creation order. */
1346  if (!grp->nc4_info->no_attr_create_order) {
1347  if (H5Pset_attr_creation_order(gcpl_id, H5P_CRT_ORDER_TRACKED|H5P_CRT_ORDER_INDEXED) < 0)
1348  BAIL(NC_EHDFERR);
1349  }
1350 
1351  /* Create the group. */
1352  if ((hdf5_grp->hdf_grpid = H5Gcreate2(parent_hdf5_grp->hdf_grpid, grp->hdr.name,
1353  H5P_DEFAULT, gcpl_id, H5P_DEFAULT)) < 0)
1354  BAIL(NC_EHDFERR);
1355 
1356 exit:
1357  if (gcpl_id > -1 && H5Pclose(gcpl_id) < 0)
1358  BAIL2(NC_EHDFERR);
1359  if (retval)
1360  if (hdf5_grp->hdf_grpid > 0 && H5Gclose(hdf5_grp->hdf_grpid) < 0)
1361  BAIL2(NC_EHDFERR);
1362  return retval;
1363 }
1364 
1377 static int
1378 attach_dimscales(NC_GRP_INFO_T *grp)
1379 {
1380  NC_VAR_INFO_T *var;
1381  NC_HDF5_VAR_INFO_T *hdf5_var;
1382  int d, v;
1383 
1384  /* Attach dimension scales. */
1385  for (v = 0; v < ncindexsize(grp->vars); v++)
1386  {
1387  /* Get pointer to var and HDF5-specific var info. */
1388  var = (NC_VAR_INFO_T *)ncindexith(grp->vars, v);
1389  assert(var && var->format_var_info);
1390  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
1391 
1392  /* Scales themselves do not attach. But I really wish they
1393  * would. */
1394  if (hdf5_var->dimscale)
1395  continue;
1396 
1397  /* Find the scale for each dimension, if any, and attach it. */
1398  for (d = 0; d < var->ndims; d++)
1399  {
1400  /* Is there a dimscale for this dimension? */
1401  if (hdf5_var->dimscale_attached)
1402  {
1403  if (!hdf5_var->dimscale_attached[d])
1404  {
1405  hid_t dsid; /* Dataset ID for dimension */
1406  assert(var->dim[d] && var->dim[d]->hdr.id == var->dimids[d] &&
1407  var->dim[d]->format_dim_info);
1408 
1409  LOG((2, "%s: attaching scale for dimid %d to var %s",
1410  __func__, var->dimids[d], var->hdr.name));
1411 
1412  /* Find dataset ID for dimension */
1413  if (var->dim[d]->coord_var)
1414  dsid = ((NC_HDF5_VAR_INFO_T *)(var->dim[d]->coord_var->format_var_info))->hdf_datasetid;
1415  else
1416  dsid = ((NC_HDF5_DIM_INFO_T *)var->dim[d]->format_dim_info)->hdf_dimscaleid;
1417  assert(dsid > 0);
1418 
1419  /* Attach the scale. */
1420  if (H5DSattach_scale(hdf5_var->hdf_datasetid, dsid, d) < 0)
1421  return NC_EDIMSCALE;
1422  hdf5_var->dimscale_attached[d] = NC_TRUE;
1423  }
1424  }
1425  }
1426  }
1427 
1428  return NC_NOERR;
1429 }
1430 
1441 static int
1442 var_exists(hid_t grpid, char *name, nc_bool_t *exists)
1443 {
1444  htri_t link_exists;
1445 
1446  /* Reset the boolean */
1447  *exists = NC_FALSE;
1448 
1449  /* Check if the object name exists in the group */
1450  if ((link_exists = H5Lexists(grpid, name, H5P_DEFAULT)) < 0)
1451  return NC_EHDFERR;
1452  if (link_exists)
1453  {
1454  H5G_stat_t statbuf;
1455 
1456  /* Get info about the object */
1457  if (H5Gget_objinfo(grpid, name, 1, &statbuf) < 0)
1458  return NC_EHDFERR;
1459 
1460  if (H5G_DATASET == statbuf.type)
1461  *exists = NC_TRUE;
1462  }
1463 
1464  return NC_NOERR;
1465 }
1466 
1482 static int
1483 remove_coord_atts(hid_t hdf_datasetid)
1484 {
1485  htri_t attr_exists;
1486 
1487  /* If the variable dataset has an optional NC_DIMID_ATT_NAME
1488  * attribute, delete it. */
1489  if ((attr_exists = H5Aexists(hdf_datasetid, NC_DIMID_ATT_NAME)) < 0)
1490  return NC_EHDFERR;
1491  if (attr_exists)
1492  {
1493  if (H5Adelete(hdf_datasetid, NC_DIMID_ATT_NAME) < 0)
1494  return NC_EHDFERR;
1495  }
1496 
1497  /* Remove the dimension scale 'CLASS' & 'NAME' attributes. */
1498  if ((attr_exists = H5Aexists(hdf_datasetid,
1499  HDF5_DIMSCALE_CLASS_ATT_NAME)) < 0)
1500  return NC_EHDFERR;
1501  if (attr_exists)
1502  {
1503  if (H5Adelete(hdf_datasetid, HDF5_DIMSCALE_CLASS_ATT_NAME) < 0)
1504  return NC_EHDFERR;
1505  }
1506  if ((attr_exists = H5Aexists(hdf_datasetid,
1507  HDF5_DIMSCALE_NAME_ATT_NAME)) < 0)
1508  return NC_EHDFERR;
1509  if (attr_exists)
1510  {
1511  if (H5Adelete(hdf_datasetid, HDF5_DIMSCALE_NAME_ATT_NAME) < 0)
1512  return NC_EHDFERR;
1513  }
1514  return NC_NOERR;
1515 }
1516 
1531 static int
1532 write_var(NC_VAR_INFO_T *var, NC_GRP_INFO_T *grp, nc_bool_t write_dimid)
1533 {
1534  NC_HDF5_GRP_INFO_T *hdf5_grp;
1535  NC_HDF5_VAR_INFO_T *hdf5_var;
1536  nc_bool_t replace_existing_var = NC_FALSE;
1537  int retval;
1538 
1539  assert(var && var->format_var_info && grp && grp->format_grp_info);
1540 
1541  LOG((4, "%s: writing var %s", __func__, var->hdr.name));
1542 
1543  /* Get HDF5-specific group and var info. */
1544  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
1545  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
1546 
1547  /* If the variable has already been created & the fill value changed,
1548  * indicate that the existing variable should be replaced. */
1549  if (var->created && var->fill_val_changed)
1550  {
1551  replace_existing_var = NC_TRUE;
1552  var->fill_val_changed = NC_FALSE;
1553  /* If the variable is going to be replaced, we need to flag any
1554  other attributes associated with the variable as 'dirty', or
1555  else *only* the fill value attribute will be copied over and
1556  the rest will be lost. See
1557  https://github.com/Unidata/netcdf-c/issues/239 */
1558  flag_atts_dirty(var->att);
1559  }
1560 
1561  /* Is this a coordinate var that has already been created in
1562  * the HDF5 file as a dimscale dataset? Check for dims with the
1563  * same name in this group. If there is one, check to see if
1564  * this object exists in the HDF group. */
1565  if (var->became_coord_var)
1566  {
1567  if ((NC_DIM_INFO_T *)ncindexlookup(grp->dim, var->hdr.name))
1568  {
1569  nc_bool_t exists;
1570 
1571  if ((retval = var_exists(hdf5_grp->hdf_grpid, var->hdr.name, &exists)))
1572  return retval;
1573  if (exists)
1574  {
1575  /* Indicate that the variable already exists, and should
1576  * be replaced. */
1577  replace_existing_var = NC_TRUE;
1578  flag_atts_dirty(var->att);
1579  }
1580  }
1581  }
1582 
1583  /* Check dims if the variable will be replaced, so that the
1584  * dimensions will be de-attached and re-attached correctly. */
1585  if (replace_existing_var)
1586  {
1587  NC_DIM_INFO_T *d1;
1588 
1589  /* Is there a dim with this var's name? */
1590  if ((d1 = (NC_DIM_INFO_T *)ncindexlookup(grp->dim, var->hdr.name)))
1591  {
1592  nc_bool_t exists;
1593  assert(d1->format_dim_info && d1->hdr.name);
1594 
1595  if ((retval = var_exists(hdf5_grp->hdf_grpid, var->hdr.name, &exists)))
1596  return retval;
1597  if (exists)
1598  {
1599  hid_t dsid;
1600 
1601  /* Find dataset ID for dimension */
1602  if (d1->coord_var)
1603  dsid = ((NC_HDF5_VAR_INFO_T *)d1->coord_var->format_var_info)->hdf_datasetid;
1604  else
1605  dsid = ((NC_HDF5_DIM_INFO_T *)d1->format_dim_info)->hdf_dimscaleid;
1606  assert(dsid > 0);
1607 
1608  /* If we're replacing an existing dimscale dataset, go to
1609  * every var in the file and detach this dimension scale,
1610  * because we have to delete it. */
1611  if ((retval = rec_detach_scales(grp->nc4_info->root_grp,
1612  var->dimids[0], dsid)))
1613  return retval;
1614  }
1615  }
1616  }
1617 
1618  /* If this is not a dimension scale, remove any attached scales,
1619  * and delete dimscale attributes from the var. */
1620  if (var->was_coord_var && hdf5_var->dimscale_attached)
1621  {
1622  int d;
1623 
1624  /* If the variable already exists in the file, Remove any dimension scale
1625  * attributes from it, if they exist. */
1626  if (var->created)
1627  if ((retval = remove_coord_atts(hdf5_var->hdf_datasetid)))
1628  return retval;
1629 
1630  /* If this is a regular var, detach all its dim scales. */
1631  for (d = 0; d < var->ndims; d++)
1632  {
1633  if (hdf5_var->dimscale_attached[d])
1634  {
1635  hid_t dsid; /* Dataset ID for dimension */
1636  assert(var->dim[d] && var->dim[d]->hdr.id == var->dimids[d] &&
1637  var->dim[d]->format_dim_info);
1638 
1639  /* Find dataset ID for dimension */
1640  if (var->dim[d]->coord_var)
1641  dsid = ((NC_HDF5_VAR_INFO_T *)var->dim[d]->coord_var->format_var_info)->hdf_datasetid;
1642  else
1643  dsid = ((NC_HDF5_DIM_INFO_T *)var->dim[d]->format_dim_info)->hdf_dimscaleid;
1644  assert(dsid > 0);
1645 
1646  /* Detach this dim scale. */
1647  if (H5DSdetach_scale(hdf5_var->hdf_datasetid, dsid, d) < 0)
1648  return NC_EHDFERR;
1649  hdf5_var->dimscale_attached[d] = NC_FALSE;
1650  }
1651  }
1652  }
1653 
1654  /* Delete the HDF5 dataset that is to be replaced. */
1655  if (replace_existing_var)
1656  {
1657  /* Free the HDF5 dataset id. */
1658  if (hdf5_var->hdf_datasetid && H5Dclose(hdf5_var->hdf_datasetid) < 0)
1659  return NC_EHDFERR;
1660  hdf5_var->hdf_datasetid = 0;
1661 
1662  /* Now delete the variable. */
1663  if (H5Gunlink(hdf5_grp->hdf_grpid, var->hdr.name) < 0)
1664  return NC_EDIMMETA;
1665  }
1666 
1667  /* Create the dataset. */
1668  if (var->is_new_var || replace_existing_var)
1669  {
1670  if ((retval = var_create_dataset(grp, var, write_dimid)))
1671  return retval;
1672  }
1673  else
1674  {
1675  if (write_dimid && var->ndims)
1676  if ((retval = write_netcdf4_dimid(hdf5_var->hdf_datasetid,
1677  var->dimids[0])))
1678  return retval;
1679  }
1680 
1681  if (replace_existing_var)
1682  {
1683  /* If this is a dimension scale, reattach the scale everywhere it
1684  * is used. (Recall that netCDF dimscales are always 1-D). */
1685  if(hdf5_var->dimscale)
1686  {
1687  if ((retval = rec_reattach_scales(grp->nc4_info->root_grp,
1688  var->dimids[0], hdf5_var->hdf_datasetid)))
1689  return retval;
1690  }
1691  /* If it's not a dimension scale, clear the dimscale attached flags,
1692  * so the dimensions are re-attached. */
1693  else
1694  {
1695  if (hdf5_var->dimscale_attached)
1696  memset(hdf5_var->dimscale_attached, 0, sizeof(nc_bool_t) * var->ndims);
1697  }
1698  }
1699 
1700  /* Clear coord. var state transition flags */
1701  var->was_coord_var = NC_FALSE;
1702  var->became_coord_var = NC_FALSE;
1703 
1704  /* Now check the attributes for this var. */
1705  if (var->attr_dirty)
1706  {
1707  /* Write attributes for this var. */
1708  if ((retval = write_attlist(var->att, var->hdr.id, grp)))
1709  return retval;
1710  var->attr_dirty = NC_FALSE;
1711  }
1712 
1713  return NC_NOERR;
1714 }
1715 
1729 int
1730 nc4_create_dim_wo_var(NC_DIM_INFO_T *dim)
1731 {
1732  NC_HDF5_DIM_INFO_T *hdf5_dim;
1733  NC_HDF5_GRP_INFO_T *hdf5_grp;
1734  hid_t spaceid = -1, create_propid = -1;
1735  hsize_t dims[1], max_dims[1], chunk_dims[1] = {1};
1736  char dimscale_wo_var[NC_MAX_NAME];
1737  int retval = NC_NOERR;
1738 
1739  LOG((4, "%s: creating dim %s", __func__, dim->hdr.name));
1740 
1741  /* Sanity check */
1742  assert(!dim->coord_var);
1743 
1744  /* Get HDF5-specific dim and group info. */
1745  hdf5_grp = (NC_HDF5_GRP_INFO_T *)dim->container->format_grp_info;
1746  hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info;
1747 
1748  /* Create a property list. */
1749  if ((create_propid = H5Pcreate(H5P_DATASET_CREATE)) < 0)
1750  BAIL(NC_EHDFERR);
1751 
1752  /* Turn off recording of times associated with this object. */
1753  if (H5Pset_obj_track_times(create_propid, 0) < 0)
1754  BAIL(NC_EHDFERR);
1755 
1756  /* Set size of dataset to size of dimension. */
1757  dims[0] = dim->len;
1758  max_dims[0] = dim->len;
1759 
1760  /* If this dimension scale is unlimited (i.e. it's an unlimited
1761  * dimension), then set up chunking, with a chunksize of 1. */
1762  if (dim->unlimited)
1763  {
1764  max_dims[0] = H5S_UNLIMITED;
1765  if (H5Pset_chunk(create_propid, 1, chunk_dims) < 0)
1766  BAIL(NC_EHDFERR);
1767  }
1768 
1769  /* Set up space. */
1770  if ((spaceid = H5Screate_simple(1, dims, max_dims)) < 0)
1771  BAIL(NC_EHDFERR);
1772 
1773  /* Turn on creation-order tracking. */
1774  if (!dim->container->nc4_info->no_attr_create_order) {
1775  if (H5Pset_attr_creation_order(create_propid, H5P_CRT_ORDER_TRACKED|
1776  H5P_CRT_ORDER_INDEXED) < 0)
1777  BAIL(NC_EHDFERR);
1778  }
1779  /* Create the dataset that will be the dimension scale. */
1780  LOG((4, "%s: about to H5Dcreate1 a dimscale dataset %s", __func__,
1781  dim->hdr.name));
1782  if ((hdf5_dim->hdf_dimscaleid = H5Dcreate2(hdf5_grp->hdf_grpid, dim->hdr.name,
1783  H5T_IEEE_F32BE, spaceid,
1784  H5P_DEFAULT, create_propid,
1785  H5P_DEFAULT)) < 0)
1786  BAIL(NC_EHDFERR);
1787 
1788  /* Indicate that this is a scale. Also indicate that not
1789  * be shown to the user as a variable. It is hidden. It is
1790  * a DIM WITHOUT A VARIABLE! */
1791  sprintf(dimscale_wo_var, "%s%10d", DIM_WITHOUT_VARIABLE, (int)dim->len);
1792  if (H5DSset_scale(hdf5_dim->hdf_dimscaleid, dimscale_wo_var) < 0)
1793  BAIL(NC_EHDFERR);
1794 
1795  /* Since this dimension was created out of order, we cannot rely on
1796  * it getting the correct dimid on file open. We must assign it
1797  * explicitly. */
1798  if ((retval = write_netcdf4_dimid(hdf5_dim->hdf_dimscaleid, dim->hdr.id)))
1799  BAIL(retval);
1800 
1801 exit:
1802  if (spaceid > 0 && H5Sclose(spaceid) < 0)
1803  BAIL2(NC_EHDFERR);
1804  if (create_propid > 0 && H5Pclose(create_propid) < 0)
1805  BAIL2(NC_EHDFERR);
1806  return retval;
1807 }
1808 
1821 static int
1822 write_dim(NC_DIM_INFO_T *dim, NC_GRP_INFO_T *grp, nc_bool_t write_dimid)
1823 {
1824  NC_HDF5_DIM_INFO_T *hdf5_dim;
1825  int retval = NC_NOERR;
1826 
1827  assert(dim && dim->format_dim_info && grp && grp->format_grp_info);
1828 
1829  /* Get HDF5-specific dim and group info. */
1830  hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info;
1831 
1832  /* If there's no dimscale dataset for this dim, create one,
1833  * and mark that it should be hidden from netCDF as a
1834  * variable. (That is, it should appear as a dimension
1835  * without an associated variable.) */
1836  if (!hdf5_dim->hdf_dimscaleid)
1837  if ((retval = nc4_create_dim_wo_var(dim)))
1838  BAIL(retval);
1839 
1840  /* Did we extend an unlimited dimension? */
1841  if (dim->extended)
1842  {
1843  NC_VAR_INFO_T *v1 = NULL;
1844 
1845  assert(dim->unlimited);
1846 
1847  /* If this is a dimension with an associated coordinate var,
1848  * then update the length of that coord var. */
1849  v1 = dim->coord_var;
1850  if (v1)
1851  {
1852  NC_HDF5_VAR_INFO_T *hdf5_v1;
1853  hsize_t *new_size;
1854  int d1;
1855 
1856  hdf5_v1 = (NC_HDF5_VAR_INFO_T *)v1->format_var_info;
1857 
1858  /* Extend the dimension scale dataset to reflect the new
1859  * length of the dimension. */
1860  if (!(new_size = malloc(v1->ndims * sizeof(hsize_t))))
1861  BAIL(NC_ENOMEM);
1862  for (d1 = 0; d1 < v1->ndims; d1++)
1863  {
1864  assert(v1->dim[d1] && v1->dim[d1]->hdr.id == v1->dimids[d1]);
1865  new_size[d1] = v1->dim[d1]->len;
1866  }
1867  if (H5Dset_extent(hdf5_v1->hdf_datasetid, new_size) < 0)
1868  BAIL(NC_EHDFERR);
1869  free(new_size);
1870  }
1871  }
1872 
1873  /* If desired, write the secret dimid. This will be used instead of
1874  * the dimid that the dimension would otherwise receive based on
1875  * creation order. This can be necessary when dims and their
1876  * coordinate variables were created in different order. */
1877  if (write_dimid && hdf5_dim->hdf_dimscaleid)
1878  if ((retval = write_netcdf4_dimid(hdf5_dim->hdf_dimscaleid, dim->hdr.id)))
1879  BAIL(retval);
1880 
1881 exit:
1882 
1883  return retval;
1884 }
1885 
1898 int
1899 nc4_rec_write_metadata(NC_GRP_INFO_T *grp, nc_bool_t bad_coord_order)
1900 {
1901  NC_DIM_INFO_T *dim = NULL;
1902  NC_VAR_INFO_T *var = NULL;
1903  NC_GRP_INFO_T *child_grp = NULL;
1904  int coord_varid = -1;
1905  int var_index = 0;
1906  int dim_index = 0;
1907  int retval;
1908  int i;
1909 
1910  assert(grp && grp->hdr.name &&
1911  ((NC_HDF5_GRP_INFO_T *)(grp->format_grp_info))->hdf_grpid);
1912  LOG((3, "%s: grp->hdr.name %s, bad_coord_order %d", __func__, grp->hdr.name,
1913  bad_coord_order));
1914 
1915  /* Write global attributes for this group. */
1916  if ((retval = write_attlist(grp->att, NC_GLOBAL, grp)))
1917  return retval;
1918 
1919  /* Set the pointers to the beginning of the list of dims & vars in this
1920  * group. */
1921  dim = (NC_DIM_INFO_T *)ncindexith(grp->dim, dim_index);
1922  var = (NC_VAR_INFO_T *)ncindexith(grp->vars, var_index);
1923 
1924  /* Because of HDF5 ordering the dims and vars have to be stored in
1925  * this way to ensure that the dims and coordinate vars come out in
1926  * the correct order. */
1927  while (dim || var)
1928  {
1929  nc_bool_t found_coord, wrote_coord;
1930 
1931  /* Write non-coord dims in order, stopping at the first one that
1932  * has an associated coord var. */
1933  for (found_coord = NC_FALSE; dim && !found_coord; )
1934  {
1935  if (!dim->coord_var)
1936  {
1937  if ((retval = write_dim(dim, grp, bad_coord_order)))
1938  return retval;
1939  }
1940  else
1941  {
1942  coord_varid = dim->coord_var->hdr.id;
1943  found_coord = NC_TRUE;
1944  }
1945  dim = (NC_DIM_INFO_T *)ncindexith(grp->dim, ++dim_index);
1946  }
1947 
1948  /* Write each var. When we get to the coord var we are waiting
1949  * for (if any), then we break after writing it. */
1950  for (wrote_coord = NC_FALSE; var && !wrote_coord; )
1951  {
1952  if ((retval = write_var(var, grp, bad_coord_order)))
1953  return retval;
1954  if (found_coord && var->hdr.id == coord_varid)
1955  wrote_coord = NC_TRUE;
1956  var = (NC_VAR_INFO_T *)ncindexith(grp->vars, ++var_index);
1957  }
1958  } /* end while */
1959 
1960  /* Attach dimscales to vars in this group. Unless directed not to. */
1961  if (!grp->nc4_info->no_dimscale_attach) {
1962  if ((retval = attach_dimscales(grp)))
1963  return retval;
1964  }
1965 
1966  /* If there are any child groups, write their metadata. */
1967  for (i = 0; i < ncindexsize(grp->children); i++)
1968  {
1969  child_grp = (NC_GRP_INFO_T *)ncindexith(grp->children, i);
1970  assert(child_grp);
1971  if ((retval = nc4_rec_write_metadata(child_grp, bad_coord_order)))
1972  return retval;
1973  }
1974  return NC_NOERR;
1975 }
1976 
1986 int
1987 nc4_rec_write_groups_types(NC_GRP_INFO_T *grp)
1988 {
1989  NC_GRP_INFO_T *child_grp;
1990  NC_HDF5_GRP_INFO_T *hdf5_grp;
1991  NC_TYPE_INFO_T *type;
1992  int retval;
1993  int i;
1994 
1995  assert(grp && grp->hdr.name && grp->format_grp_info);
1996  LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
1997 
1998  /* Get HDF5-specific group info. */
1999  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
2000 
2001  /* Create the group in the HDF5 file if it doesn't exist. */
2002  if (!hdf5_grp->hdf_grpid)
2003  if ((retval = create_group(grp)))
2004  return retval;
2005 
2006  /* If this is the root group of a file with strict NC3 rules, write
2007  * an attribute. But don't leave the attribute open. */
2008  if (!grp->parent && (grp->nc4_info->cmode & NC_CLASSIC_MODEL))
2009  if ((retval = write_nc3_strict_att(hdf5_grp->hdf_grpid)))
2010  return retval;
2011 
2012  /* If there are any user-defined types, write them now. */
2013  for(i=0;i<ncindexsize(grp->type);i++) {
2014  type = (NC_TYPE_INFO_T *)ncindexith(grp->type, i);
2015  assert(type);
2016  if ((retval = commit_type(grp, type)))
2017  return retval;
2018  }
2019 
2020  /* If there are any child groups, write their groups and types. */
2021  for(i=0;i<ncindexsize(grp->children);i++) {
2022  if((child_grp = (NC_GRP_INFO_T*)ncindexith(grp->children,i)) == NULL) continue;
2023  if ((retval = nc4_rec_write_groups_types(child_grp)))
2024  return retval;
2025  }
2026  return NC_NOERR;
2027 }
2028 
2042 int
2043 nc4_rec_match_dimscales(NC_GRP_INFO_T *grp)
2044 {
2045  NC_GRP_INFO_T *g;
2046  NC_VAR_INFO_T *var;
2047  NC_DIM_INFO_T *dim;
2048  int retval = NC_NOERR;
2049  int i;
2050 
2051  assert(grp && grp->hdr.name);
2052  LOG((4, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
2053 
2054  /* Perform var dimscale match for child groups. */
2055  for (i = 0; i < ncindexsize(grp->children); i++)
2056  {
2057  g = (NC_GRP_INFO_T*)ncindexith(grp->children, i);
2058  assert(g);
2059  if ((retval = nc4_rec_match_dimscales(g)))
2060  return retval;
2061  }
2062 
2063  /* Check all the vars in this group. If they have dimscale info,
2064  * try and find a dimension for them. */
2065  for (i = 0; i < ncindexsize(grp->vars); i++)
2066  {
2067  NC_HDF5_VAR_INFO_T *hdf5_var;
2068  int ndims;
2069  int d;
2070 
2071  /* Get pointer to var and to the HDF5-specific var info. */
2072  var = (NC_VAR_INFO_T*)ncindexith(grp->vars, i);
2073  assert(var && var->format_var_info);
2074  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
2075 
2076  /* Check all vars and see if dim[i] != NULL if dimids[i] valid. */
2077  /* This loop is very odd. Under normal circumstances, var->dimid[d] is zero
2078  (from the initial calloc) which is a legitimate dimid. The code does not
2079  distinquish this case from the dimscale case where the id might actually
2080  be defined.
2081  The original nc4_find_dim searched up the group tree looking for the given
2082  dimid in one of the dim lists associated with each ancestor group.
2083  I changed nc4_fnd_dim to use the dimid directly using h5->alldims.
2084  However, here that is incorrect because it will find the dimid 0 always
2085  (if any dimensions were defined). Except that when dimscale dimids have
2086  been defined, one or more of the values in var->dimids will have a
2087  legitimate value.
2088  The solution I choose is to modify nc4_var_list_add to initialize dimids to
2089  illegal values (-1). This is another example of the problems with dimscales.
2090  */
2091  ndims = var->ndims;
2092  for (d = 0; d < ndims; d++)
2093  {
2094  if (var->dim[d] == NULL) {
2095  nc4_find_dim(grp, var->dimids[d], &var->dim[d], NULL);
2096  }
2097  /* assert(var->dim[d] && var->dim[d]->hdr.id == var->dimids[d]); */
2098  }
2099 
2100  /* Skip dimension scale variables */
2101  if (!hdf5_var->dimscale)
2102  {
2103  int d;
2104  int j;
2105 
2106  /* Are there dimscales for this variable? */
2107  if (hdf5_var->dimscale_hdf5_objids)
2108  {
2109  for (d = 0; d < var->ndims; d++)
2110  {
2111  nc_bool_t finished = NC_FALSE;
2112  LOG((5, "%s: var %s has dimscale info...", __func__, var->hdr.name));
2113 
2114  /* Check this and parent groups. */
2115  for (g = grp; g && !finished; g = g->parent)
2116  {
2117  /* Check all dims in this group. */
2118  for (j = 0; j < ncindexsize(g->dim); j++)
2119  {
2120  /* Get the HDF5 specific dim info. */
2121  NC_HDF5_DIM_INFO_T *hdf5_dim;
2122  dim = (NC_DIM_INFO_T *)ncindexith(g->dim, j);
2123  assert(dim && dim->format_dim_info);
2124  hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info;
2125 
2126  /* Check for exact match of fileno/objid arrays
2127  * to find identical objects in HDF5 file. */
2128 #if H5_VERSION_GE(1,12,0)
2129  int token_cmp;
2130  if (H5Otoken_cmp(hdf5_var->hdf_datasetid, &hdf5_var->dimscale_hdf5_objids[d].token, &hdf5_dim->hdf5_objid.token, &token_cmp) < 0)
2131  return NC_EHDFERR;
2132 
2133  if (hdf5_var->dimscale_hdf5_objids[d].fileno == hdf5_dim->hdf5_objid.fileno &&
2134  token_cmp == 0)
2135 #else
2136  if (hdf5_var->dimscale_hdf5_objids[d].fileno[0] == hdf5_dim->hdf5_objid.fileno[0] &&
2137  hdf5_var->dimscale_hdf5_objids[d].objno[0] == hdf5_dim->hdf5_objid.objno[0] &&
2138  hdf5_var->dimscale_hdf5_objids[d].fileno[1] == hdf5_dim->hdf5_objid.fileno[1] &&
2139  hdf5_var->dimscale_hdf5_objids[d].objno[1] == hdf5_dim->hdf5_objid.objno[1])
2140 #endif
2141  {
2142  LOG((4, "%s: for dimension %d, found dim %s", __func__,
2143  d, dim->hdr.name));
2144  var->dimids[d] = dim->hdr.id;
2145  var->dim[d] = dim;
2146  finished = NC_TRUE;
2147  break;
2148  }
2149  } /* next dim */
2150  } /* next grp */
2151  LOG((5, "%s: dimid for this dimscale is %d", __func__,
2152  var->type_info->hdr.id));
2153  } /* next var->dim */
2154  }
2155  /* No dimscales for this var! Invent phony dimensions. */
2156  else
2157  {
2158  hid_t spaceid = 0;
2159  hsize_t *h5dimlen = NULL, *h5dimlenmax = NULL;
2160  int dataset_ndims;
2161 
2162  /* Find the space information for this dimension. */
2163  if ((spaceid = H5Dget_space(hdf5_var->hdf_datasetid)) < 0)
2164  return NC_EHDFERR;
2165 
2166  /* Get the len of each dim in the space. */
2167  if (var->ndims)
2168  {
2169  if (!(h5dimlen = malloc(var->ndims * sizeof(hsize_t))))
2170  return NC_ENOMEM;
2171  if (!(h5dimlenmax = malloc(var->ndims * sizeof(hsize_t))))
2172  {
2173  free(h5dimlen);
2174  return NC_ENOMEM;
2175  }
2176  if ((dataset_ndims = H5Sget_simple_extent_dims(spaceid, h5dimlen,
2177  h5dimlenmax)) < 0) {
2178  free(h5dimlenmax);
2179  free(h5dimlen);
2180  return NC_EHDFERR;
2181  }
2182  if (dataset_ndims != var->ndims) {
2183  free(h5dimlenmax);
2184  free(h5dimlen);
2185  return NC_EHDFERR;
2186  }
2187  }
2188  else
2189  {
2190  /* Make sure it's scalar. */
2191  if (H5Sget_simple_extent_type(spaceid) != H5S_SCALAR)
2192  return NC_EHDFERR;
2193  }
2194 
2195  /* Release the space object. */
2196  if (H5Sclose(spaceid) < 0) {
2197  free(h5dimlen);
2198  free(h5dimlenmax);
2199  return NC_EHDFERR;
2200  }
2201 
2202  /* Create a phony dimension for each dimension in the
2203  * dataset, unless there already is one the correct
2204  * size. */
2205  for (d = 0; d < var->ndims; d++)
2206  {
2207  int k;
2208  int match;
2209  /* Is there already a phony dimension of the correct size? */
2210  for(match=-1,k=0;k<ncindexsize(grp->dim);k++) {
2211  if((dim = (NC_DIM_INFO_T*)ncindexith(grp->dim,k)) == NULL) continue;
2212  if ((dim->len == h5dimlen[d]) &&
2213  ((h5dimlenmax[d] == H5S_UNLIMITED && dim->unlimited) ||
2214  (h5dimlenmax[d] != H5S_UNLIMITED && !dim->unlimited)))
2215  {match = k; break;}
2216  }
2217 
2218  /* Didn't find a phony dim? Then create one. */
2219  if (match < 0)
2220  {
2221  char phony_dim_name[NC_MAX_NAME + 1];
2222  sprintf(phony_dim_name, "phony_dim_%d", grp->nc4_info->next_dimid);
2223  LOG((3, "%s: creating phony dim for var %s", __func__, var->hdr.name));
2224  if ((retval = nc4_dim_list_add(grp, phony_dim_name, h5dimlen[d], -1, &dim)))
2225  {
2226  free(h5dimlenmax);
2227  free(h5dimlen);
2228  return retval;
2229  }
2230  /* Create struct for HDF5-specific dim info. */
2231  if (!(dim->format_dim_info = calloc(1, sizeof(NC_HDF5_DIM_INFO_T))))
2232  return NC_ENOMEM;
2233  if (h5dimlenmax[d] == H5S_UNLIMITED)
2234  dim->unlimited = NC_TRUE;
2235  }
2236 
2237  /* The variable must remember the dimid. */
2238  var->dimids[d] = dim->hdr.id;
2239  var->dim[d] = dim;
2240  } /* next dim */
2241 
2242  /* Free the memory we malloced. */
2243  free(h5dimlen);
2244  free(h5dimlenmax);
2245  }
2246  }
2247  }
2248 
2249  return retval;
2250 }
2251 
2263 void
2264 reportobject(int uselog, hid_t id, unsigned int type)
2265 {
2266  char name[NC_HDF5_MAX_NAME];
2267  ssize_t len;
2268  const char* typename = NULL;
2269  long long printid = (long long)id;
2270 
2271  len = H5Iget_name(id, name, NC_HDF5_MAX_NAME);
2272  if(len < 0) return;
2273  name[len] = '\0';
2274 
2275  switch (type) {
2276  case H5F_OBJ_FILE: typename = "File"; break;
2277  case H5F_OBJ_DATASET: typename = "Dataset"; break;
2278  case H5F_OBJ_GROUP: typename = "Group"; break;
2279  case H5F_OBJ_DATATYPE: typename = "Datatype"; break;
2280  case H5F_OBJ_ATTR:
2281  typename = "Attribute";
2282  len = H5Aget_name(id, NC_HDF5_MAX_NAME, name);
2283  if(len < 0) len = 0;
2284  name[len] = '\0';
2285  break;
2286  default: typename = "<unknown>"; break;
2287  }
2288 #ifdef LOGGING
2289  if(uselog) {
2290  LOG((0,"Type = %s(%lld) name='%s'",typename,printid,name));
2291  } else
2292 #endif
2293  {
2294  fprintf(stderr,"Type = %s(%lld) name='%s'",typename,printid,name);
2295  }
2296 
2297 }
2298 
2309 static void
2310 reportopenobjectsT(int uselog, hid_t fid, int ntypes, unsigned int* otypes)
2311 {
2312  int t,i;
2313  ssize_t ocount;
2314  size_t maxobjs = -1;
2315  hid_t* idlist = NULL;
2316 
2317  /* Always report somehow */
2318 #ifdef LOGGING
2319  if(uselog)
2320  LOG((0,"\nReport: open objects on %lld",(long long)fid));
2321  else
2322 #endif
2323  fprintf(stdout,"\nReport: open objects on %lld\n",(long long)fid);
2324  maxobjs = H5Fget_obj_count(fid,H5F_OBJ_ALL);
2325  if(idlist != NULL) free(idlist);
2326  idlist = (hid_t*)malloc(sizeof(hid_t)*maxobjs);
2327  for(t=0;t<ntypes;t++) {
2328  unsigned int ot = otypes[t];
2329  ocount = H5Fget_obj_ids(fid,ot,maxobjs,idlist);
2330  for(i=0;i<ocount;i++) {
2331  hid_t o = idlist[i];
2332  reportobject(uselog,o,ot);
2333  }
2334  }
2335  if(idlist != NULL) free(idlist);
2336 }
2337 
2346 void
2347 reportopenobjects(int uselog, hid_t fid)
2348 {
2349  unsigned int OTYPES[5] = {H5F_OBJ_FILE, H5F_OBJ_DATASET, H5F_OBJ_GROUP,
2350  H5F_OBJ_DATATYPE, H5F_OBJ_ATTR};
2351 
2352  reportopenobjectsT(uselog, fid ,5, OTYPES);
2353 }
2354 
2362 void
2363 showopenobjects5(NC_FILE_INFO_T* h5)
2364 {
2365  NC_HDF5_FILE_INFO_T *hdf5_info;
2366 
2367  assert(h5 && h5->format_file_info);
2368  hdf5_info = (NC_HDF5_FILE_INFO_T *)h5->format_file_info;
2369 
2370  fprintf(stderr,"===== begin showopenobjects =====\n");
2371  reportopenobjects(0,hdf5_info->hdfid);
2372  fprintf(stderr,"===== end showopenobjects =====\n");
2373  fflush(stderr);
2374 }
2375 
2384 void
2385 showopenobjects(int ncid)
2386 {
2387  NC_FILE_INFO_T* h5 = NULL;
2388 
2389  /* Find our metadata for this file. */
2390  if (nc4_find_nc_grp_h5(ncid, NULL, NULL, &h5) != NC_NOERR)
2391  fprintf(stderr,"failed\n");
2392  else
2393  showopenobjects5(h5);
2394  fflush(stderr);
2395 }
2396 
2408 int
2409 NC4_hdf5get_libversion(unsigned* major,unsigned* minor,unsigned* release)
2410 {
2411  if(H5get_libversion(major,minor,release) < 0)
2412  return NC_EHDFERR;
2413  return NC_NOERR;
2414 }
2415 
2426 int
2427 NC4_hdf5get_superblock(struct NC_FILE_INFO* h5, int* idp)
2428 {
2429  NC_HDF5_FILE_INFO_T *hdf5_info;
2430  int stat = NC_NOERR;
2431  unsigned super;
2432  hid_t plist = -1;
2433 
2434  assert(h5 && h5->format_file_info);
2435  hdf5_info = (NC_HDF5_FILE_INFO_T *)h5->format_file_info;
2436 
2437  if((plist = H5Fget_create_plist(hdf5_info->hdfid)) < 0)
2438  {stat = NC_EHDFERR; goto done;}
2439  if(H5Pget_version(plist, &super, NULL, NULL, NULL) < 0)
2440  {stat = NC_EHDFERR; goto done;}
2441  if(idp) *idp = (int)super;
2442 done:
2443  if(plist >= 0) H5Pclose(plist);
2444  return stat;
2445 }
2446 
2447 static int NC4_strict_att_exists(NC_FILE_INFO_T*);
2448 static int NC4_walk(hid_t, int*);
2449 
2475 int
2476 NC4_isnetcdf4(struct NC_FILE_INFO* h5)
2477 {
2478  int stat;
2479  int isnc4 = 0;
2480  int exists;
2481  int count;
2482 
2483  /* Look for NC3_STRICT_ATT_NAME */
2484  exists = NC4_strict_att_exists(h5);
2485  if(exists)
2486  goto done;
2487  /* attribute did not exist */
2488  /* => last resort: walk the HDF5 file looking for markers */
2489  count = 0;
2490  stat = NC4_walk(((NC_HDF5_GRP_INFO_T *)(h5->root_grp->format_grp_info))->hdf_grpid,
2491  &count);
2492  if(stat != NC_NOERR)
2493  isnc4 = 0;
2494  else /* Threshold is at least two matches */
2495  isnc4 = (count >= 2);
2496 
2497 done:
2498  return isnc4;
2499 }
2500 
2509 static int
2510 NC4_strict_att_exists(NC_FILE_INFO_T *h5)
2511 {
2512  hid_t grpid = -1;
2513  htri_t attr_exists;
2514 
2515  /* Get root group ID. */
2516  grpid = ((NC_HDF5_GRP_INFO_T *)(h5->root_grp->format_grp_info))->hdf_grpid;
2517 
2518  /* See if the NC3_STRICT_ATT_NAME attribute exists */
2519  if ((attr_exists = H5Aexists(grpid, NC3_STRICT_ATT_NAME)) < 0)
2520  return 1;
2521  return (attr_exists?1:0);
2522 }
2523 
2533 static int
2534 NC4_walk(hid_t gid, int* countp)
2535 {
2536  int ncstat = NC_NOERR;
2537  int i,j,na;
2538  ssize_t len;
2539  hsize_t nobj;
2540  herr_t err;
2541  int otype;
2542  hid_t grpid, dsid;
2543  char name[NC_HDF5_MAX_NAME];
2544 
2545  /* walk group members of interest */
2546  err = H5Gget_num_objs(gid, &nobj);
2547  if(err < 0) return err;
2548 
2549  for(i = 0; i < nobj; i++) {
2550  /* Get name & kind of object in the group */
2551  len = H5Gget_objname_by_idx(gid,(hsize_t)i,name,(size_t)NC_HDF5_MAX_NAME);
2552  if(len < 0) return len;
2553 
2554  otype = H5Gget_objtype_by_idx(gid,(size_t)i);
2555  switch(otype) {
2556  case H5G_GROUP:
2557  grpid = H5Gopen1(gid,name);
2558  NC4_walk(grpid,countp);
2559  H5Gclose(grpid);
2560  break;
2561  case H5G_DATASET: /* variables */
2562  /* Check for phony_dim */
2563  if(strcmp(name,"phony_dim")==0)
2564  *countp = *countp + 1;
2565  dsid = H5Dopen1(gid,name);
2566  na = H5Aget_num_attrs(dsid);
2567  for(j = 0; j < na; j++) {
2568  hid_t aid = H5Aopen_idx(dsid,(unsigned int) j);
2569  if(aid >= 0) {
2570  const NC_reservedatt* ra;
2571  ssize_t len = H5Aget_name(aid, NC_HDF5_MAX_NAME, name);
2572  if(len < 0) return len;
2573  /* Is this a netcdf-4 marker attribute */
2574  /* Is this a netcdf-4 marker attribute */
2575  ra = NC_findreserved(name);
2576  if(ra != NULL)
2577  *countp = *countp + 1;
2578  }
2579  H5Aclose(aid);
2580  }
2581  H5Dclose(dsid);
2582  break;
2583  default:/* ignore */
2584  break;
2585  }
2586  }
2587  return ncstat;
2588 }
2589 
EXTERNL int nc_free_vlen(nc_vlen_t *vl)
Free memory in a VLEN object.
Definition: dvlen.c:43
Main header file for the C API.
#define NC_EBADTYPE
Not a netcdf data type.
Definition: netcdf.h:410
#define NC_UINT
unsigned 4-byte int
Definition: netcdf.h:44
#define NC_ENDIAN_NATIVE
In HDF5 files you can set the endianness of variables with nc_def_var_endian().
Definition: netcdf.h:294
#define NC_EFILTER
Filter operation failed.
Definition: netcdf.h:513
#define NC_QUANTIZE_GRANULARBR_ATT_NAME
When quantization is used for a variable, an attribute of the appropriate name is added.
Definition: netcdf.h:344
#define NC_INT
signed 4 byte integer
Definition: netcdf.h:38
#define NC_ENDIAN_BIG
In HDF5 files you can set the endianness of variables with nc_def_var_endian().
Definition: netcdf.h:296
#define NC_MAX_VAR_DIMS
max per variable dimensions
Definition: netcdf.h:282
#define NC_BYTE
signed 1 byte integer
Definition: netcdf.h:35
#define NC_EPERM
Write to read only.
Definition: netcdf.h:379
#define NC_QUANTIZE_BITROUND
Use BitRound quantization.
Definition: netcdf.h:338
#define NC_EDIMSCALE
Problem with HDF5 dimscales.
Definition: netcdf.h:504
#define NC_VLEN
vlen (variable-length) types
Definition: netcdf.h:53
#define NC_NAT
Not A Type.
Definition: netcdf.h:34
#define NC_DOUBLE
double precision floating point number
Definition: netcdf.h:41
#define NC_UBYTE
unsigned 1 byte int
Definition: netcdf.h:42
#define NC_FLOAT
single precision floating point number
Definition: netcdf.h:40
#define NC_ENOMEM
Memory allocation (malloc) failure.
Definition: netcdf.h:448
#define NC_COMPOUND
compound types
Definition: netcdf.h:56
#define NC_CHUNKED
In HDF5 files you can set storage for each variable to be either contiguous or chunked,...
Definition: netcdf.h:304
#define NC_SHORT
signed 2 byte integer
Definition: netcdf.h:37
#define NC_QUANTIZE_GRANULARBR
Use Granular BitRound quantization.
Definition: netcdf.h:337
#define NC_QUANTIZE_BITGROOM
Use BitGroom quantization.
Definition: netcdf.h:336
#define NC_ENUM
enum types
Definition: netcdf.h:55
#define NC_INT64
signed 8-byte int
Definition: netcdf.h:45
#define NC_EATTMETA
Problem with attribute metadata.
Definition: netcdf.h:487
#define NC_EHDFERR
Error at HDF5 layer.
Definition: netcdf.h:481
#define NC_CONTIGUOUS
In HDF5 files you can set storage for each variable to be either contiguous or chunked,...
Definition: netcdf.h:305
#define NC_GLOBAL
Attribute id to put/get a global attribute.
Definition: netcdf.h:254
#define NC_UINT64
unsigned 8-byte int
Definition: netcdf.h:46
#define NC_COMPACT
In HDF5 files you can set storage for each variable to be either contiguous or chunked,...
Definition: netcdf.h:306
#define NC_EFILEMETA
Problem with file metadata.
Definition: netcdf.h:485
#define NC_ENOTVAR
Variable not found.
Definition: netcdf.h:422
#define NC_EINVAL
Invalid Argument.
Definition: netcdf.h:378
#define NC_CLASSIC_MODEL
Enforce classic model on netCDF-4.
Definition: netcdf.h:140
#define NC_ENDIAN_LITTLE
In HDF5 files you can set the endianness of variables with nc_def_var_endian().
Definition: netcdf.h:295
#define NC_MAX_NAME
Maximum for classic library.
Definition: netcdf.h:281
#define NC_NOERR
No Error.
Definition: netcdf.h:368
#define NC_EDIMMETA
Problem with dimension metadata.
Definition: netcdf.h:486
#define NC_USHORT
unsigned 2-byte int
Definition: netcdf.h:43
#define NC_OPAQUE
opaque types
Definition: netcdf.h:54
#define NC_STRING
string
Definition: netcdf.h:47
#define NC_QUANTIZE_BITGROOM_ATT_NAME
When quantization is used for a variable, an attribute of the appropriate name is added.
Definition: netcdf.h:343
#define NC_CHAR
ISO/ASCII character.
Definition: netcdf.h:36
#define NC_EVARMETA
Problem with variable metadata.
Definition: netcdf.h:488
#define NC_QUANTIZE_BITROUND_ATT_NAME
When quantization is used for a variable, an attribute of the appropriate name is added.
Definition: netcdf.h:345
int nc_type
The nc_type type is just an int.
Definition: netcdf.h:25
This is the type of arrays of vlens.
Definition: netcdf.h:746