11 #include "nc4internal.h"
13 #include <H5DSpublic.h>
14 #include "nc4dispatch.h"
15 #include "ncdispatch.h"
28 extern int num_plists;
29 extern int num_spaces;
32 #define MIN_DEFLATE_LEVEL 0
33 #define MAX_DEFLATE_LEVEL 9
37 #define REFERENCE_LIST "REFERENCE_LIST"
39 #define DIMENSION_LIST "DIMENSION_LIST"
43 static int NC4_enddef(
int ncid);
44 static int nc4_rec_read_types(NC_GRP_INFO_T *grp);
45 static int nc4_rec_read_vars(NC_GRP_INFO_T *grp);
50 extern NC_FILE_INFO_T *nc_file;
55 size_t nc4_chunk_cache_size = CHUNK_CACHE_SIZE;
56 size_t nc4_chunk_cache_nelems = CHUNK_CACHE_NELEMS;
57 float nc4_chunk_cache_preemption = CHUNK_CACHE_PREEMPTION;
60 extern int default_create_format;
64 static int virgin = 1;
69 static hid_t native_type_constant[NUM_TYPES];
71 static char nc_type_name[NUM_TYPES][
NC_MAX_NAME + 1] = {
"char",
"byte",
"short",
"int",
"float",
72 "double",
"ubyte",
"ushort",
"uint",
73 "int64",
"uint64",
"string"};
74 int nc4_free_global_hdf_string_typeid();
79 nc_set_chunk_cache(
size_t size,
size_t nelems,
float preemption)
81 if (preemption < 0 || preemption > 1)
83 nc4_chunk_cache_size = size;
84 nc4_chunk_cache_nelems = nelems;
85 nc4_chunk_cache_preemption = preemption;
92 nc_get_chunk_cache(
size_t *sizep,
size_t *nelemsp,
float *preemptionp)
95 *sizep = nc4_chunk_cache_size;
98 *nelemsp = nc4_chunk_cache_nelems;
101 *preemptionp = nc4_chunk_cache_preemption;
107 nc_set_chunk_cache_ints(
int size,
int nelems,
int preemption)
109 if (size <= 0 || nelems <= 0 || preemption < 0 || preemption > 100)
111 nc4_chunk_cache_size = size;
112 nc4_chunk_cache_nelems = nelems;
113 nc4_chunk_cache_preemption = (float)preemption / 100;
118 nc_get_chunk_cache_ints(
int *sizep,
int *nelemsp,
int *preemptionp)
121 *sizep = (int)nc4_chunk_cache_size;
123 *nelemsp = (int)nc4_chunk_cache_nelems;
125 *preemptionp = (int)(nc4_chunk_cache_preemption * 100);
155 #define MAGIC_NUMBER_LEN 4
156 #define NC_HDF5_FILE 1
157 #define NC_HDF4_FILE 2
159 nc_check_for_hdf(
const char *path,
int use_parallel, MPI_Comm comm, MPI_Info info,
162 char blob[MAGIC_NUMBER_LEN];
164 assert(hdf_file && path);
165 LOG((3,
"nc_check_for_hdf: path %s", path));
175 if ((retval = MPI_File_open(comm, (
char *)path, MPI_MODE_RDONLY,
176 info, &fh)) != MPI_SUCCESS)
178 if ((retval = MPI_File_read(fh, blob, MAGIC_NUMBER_LEN, MPI_CHAR,
179 &status)) != MPI_SUCCESS)
181 if ((retval = MPI_File_close(&fh)) != MPI_SUCCESS)
188 if (!(fp = fopen(path,
"r")) ||
189 fread(blob, MAGIC_NUMBER_LEN, 1, fp) != 1)
195 if (blob[1] ==
'H' && blob[2] ==
'D' && blob[3] ==
'F')
196 *hdf_file = NC_HDF5_FILE;
197 else if (!strncmp(blob,
"\016\003\023\001", MAGIC_NUMBER_LEN))
198 *hdf_file = NC_HDF4_FILE;
208 nc4_create_file(
const char *path,
int cmode, MPI_Comm comm, MPI_Info info,
211 hid_t fcpl_id, fapl_id;
218 flags = H5F_ACC_TRUNC;
220 flags = H5F_ACC_EXCL;
222 flags = H5F_ACC_TRUNC;
224 LOG((3,
"nc4_create_file: path %s mode 0x%x", path, cmode));
230 if (cmode & NC_DISKLESS) {
233 }
else if ((cmode & NC_NOCLOBBER) && (fp = fopen(path,
"r"))) {
239 if ((retval = nc4_nc4f_list_add(nc, path, (
NC_WRITE | cmode))))
241 assert(nc->nc4_info && nc->nc4_info->root_grp);
246 if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
252 if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_SEMI))
255 if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_STRONG))
264 nc->nc4_info->parallel++;
265 if (cmode & NC_MPIIO)
267 LOG((4,
"creating parallel file with MPI/IO"));
268 if (H5Pset_fapl_mpio(fapl_id, comm, info) < 0)
273 LOG((4,
"creating parallel file with MPI/posix"));
274 if (H5Pset_fapl_mpiposix(fapl_id, comm, 0) < 0)
279 if(cmode & NC_DISKLESS) {
280 if (H5Pset_fapl_core(fapl_id, 4096, persist))
283 if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size,
284 nc4_chunk_cache_preemption) < 0)
286 LOG((4,
"nc4_create_file: set HDF raw chunk cache to size %d nelems %d preemption %f",
287 nc4_chunk_cache_size, nc4_chunk_cache_nelems, nc4_chunk_cache_preemption));
290 if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_18, H5F_LIBVER_18) < 0)
294 if ((fcpl_id = H5Pcreate(H5P_FILE_CREATE)) < 0)
303 if (H5Pset_link_creation_order(fcpl_id, (H5P_CRT_ORDER_TRACKED |
304 H5P_CRT_ORDER_INDEXED)) < 0)
306 if (H5Pset_attr_creation_order(fcpl_id, (H5P_CRT_ORDER_TRACKED |
307 H5P_CRT_ORDER_INDEXED)) < 0)
311 if ((nc->nc4_info->hdfid = H5Fcreate(path, flags, fcpl_id, fapl_id)) < 0)
315 if ((nc->nc4_info->root_grp->hdf_grpid = H5Gopen2(nc->nc4_info->hdfid,
"/",
320 if (H5Pclose(fapl_id) < 0 ||
321 H5Pclose(fcpl_id) < 0)
329 nc->nc4_info->flags |= NC_INDEF;
334 if (nc->nc4_info->hdfid > 0) H5Fclose(nc->nc4_info->hdfid);
354 NC4_create(
const char* path,
int cmode,
size_t initialsz,
int basepe,
355 size_t *chunksizehintp,
int use_parallel,
void *mpidata,
356 NC_Dispatch *dispatch, NC **ncpp)
358 NC_FILE_INFO_T *nc_file = NULL;
363 int comm = 0, info = 0;
367 assert(ncpp && path);
369 LOG((1,
"nc4_create_file: path %s cmode 0x%x comm %d info %d",
370 path, cmode, comm, info));
375 comm = ((NC_MPI_INFO *)mpidata)->comm;
376 info = ((NC_MPI_INFO *)mpidata)->info;
383 if (H5Eset_auto(NULL, NULL) < 0)
384 LOG((0,
"Couldn't turn off HDF5 error messages!"));
385 LOG((1,
"HDF5 error messages have been turned off."));
396 || (cmode & NC_MPIIO && cmode & NC_MPIPOSIX)
398 || (cmode & (NC_MPIIO | NC_MPIPOSIX) && cmode & NC_DISKLESS)
405 if ((res = nc4_file_list_add(&nc_file, dispatch)))
418 LOG((2,
"cmode after applying default format: 0x%x", cmode));
422 if (cmode & NC_NETCDF4)
424 nc_file->int_ncid = nc_file->ext_ncid;
425 res = nc4_create_file(path, cmode, comm, info, nc_file);
430 nc_file->pnetcdf_file++;
431 res = ncmpi_create(comm, path, cmode, info, &(nc_file->int_ncid));
443 nc4_file_list_del(nc_file);
446 *ncpp = (NC *)nc_file;
457 read_scale(NC_GRP_INFO_T *grp, hid_t datasetid,
char *obj_name,
458 hsize_t scale_size, hsize_t max_scale_size,
459 int *dim_without_var)
468 if ((retval = nc4_dim_list_add(&grp->dim)))
472 grp->dim->dimid = grp->file->nc4_info->next_dimid++;
478 if ((attid = H5Aopen_by_name(datasetid,
".", NC_DIMID_ATT_NAME,
479 H5P_DEFAULT, H5P_DEFAULT)) > 0)
481 if (H5Aread(attid, H5T_NATIVE_INT, &grp->dim->dimid) < 0)
483 if (H5Aclose(attid) < 0)
489 if (!(grp->dim->name = malloc((max_len + 1) *
sizeof(
char))))
491 strncpy(grp->dim->name, obj_name, max_len + 1);
495 grp->dim->too_long = 1;
498 grp->dim->len = scale_size;
499 grp->dim->hdf_dimscaleid = datasetid;
503 if (max_scale_size == H5S_UNLIMITED)
504 grp->dim->unlimited++;
509 if (H5DSget_scale_name(datasetid, dimscale_name_att,
NC_MAX_NAME) >= 0)
511 if (!strncmp(dimscale_name_att, DIM_WITHOUT_VARIABLE,
512 strlen(DIM_WITHOUT_VARIABLE)))
514 if (grp->dim->unlimited)
516 size_t len = 0, *lenp = &len;
517 if ((retval = nc4_find_dim_len(grp, grp->dim->dimid, &lenp)))
519 grp->dim->len = *lenp;
521 (*dim_without_var)++;
531 read_coord_dimids(NC_VAR_INFO_T *var)
533 hid_t coord_att_typeid = -1, coord_attid = -1, spaceid = -1;
534 hssize_t coord_array_size;
540 if ((coord_attid = H5Aopen_name(var->hdf_datasetid, COORDINATES)) < 0) ret++;
541 if (!ret && (coord_att_typeid = H5Aget_type(coord_attid)) < 0) ret++;
542 if (!ret && H5Aread(coord_attid, coord_att_typeid, var->dimids) < 0) ret++;
543 LOG((4,
"dimscale %s is multidimensional and has coords", var->name));
546 if ((spaceid = H5Aget_space(coord_attid)) < 0) ret++;
550 if ((coord_array_size = H5Sget_simple_extent_npoints(spaceid)) < 0) ret++;
556 if (spaceid >= 0 && H5Sclose(spaceid) < 0) ret++;
560 if (coord_att_typeid >= 0 && H5Tclose(coord_att_typeid) < 0) ret++;
561 if (coord_attid >= 0 && H5Aclose(coord_attid) < 0) ret++;
568 dimscale_visitor(hid_t did,
unsigned dim, hid_t dsid,
569 void *dimscale_hdf5_objids)
574 if (H5Gget_objinfo(dsid,
".", 1, &statbuf) < 0)
580 (*(HDF5_OBJID_T *)dimscale_hdf5_objids).fileno[0] = statbuf.fileno[0];
581 (*(HDF5_OBJID_T *)dimscale_hdf5_objids).fileno[1] = statbuf.fileno[1];
582 (*(HDF5_OBJID_T *)dimscale_hdf5_objids).objno[0] = statbuf.objno[0];
583 (*(HDF5_OBJID_T *)dimscale_hdf5_objids).objno[1] = statbuf.objno[1];
589 get_netcdf_type(NC_HDF5_FILE_INFO_T *h5, hid_t native_typeid,
592 NC_TYPE_INFO_T *type;
594 htri_t is_str, equal = 0;
598 if ((
class = H5Tget_class(native_typeid)) < 0)
603 if (
class == H5T_STRING)
605 if ((is_str = H5Tis_variable_str(native_typeid)) < 0)
613 else if (
class == H5T_INTEGER ||
class == H5T_FLOAT)
617 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_SCHAR)) < 0)
624 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_SHORT)) < 0)
631 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_INT)) < 0)
638 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_FLOAT)) < 0)
645 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_DOUBLE)) < 0)
652 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_UCHAR)) < 0)
659 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_USHORT)) < 0)
666 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_UINT)) < 0)
673 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_LLONG)) < 0)
680 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_ULLONG)) < 0)
691 if((type = nc4_rec_find_hdf_type(h5->root_grp, native_typeid)))
693 *xtype = type->nc_typeid;
705 get_type_info2(NC_HDF5_FILE_INFO_T *h5, hid_t datasetid,
706 nc_type *xtype, NC_TYPE_INFO_T **type_info)
708 NC_TYPE_INFO_T *type;
709 htri_t is_str, equal = 0;
710 hid_t
class, native_typeid, hdf_typeid;
719 int type_size[NUM_TYPES] = {
sizeof(char),
sizeof(
char),
sizeof(short),
720 sizeof(
int),
sizeof(float),
sizeof(
double),
721 sizeof(
unsigned char),
sizeof(
unsigned short),
722 sizeof(
unsigned int),
sizeof(
long long),
723 sizeof(
unsigned long long), 0};
726 assert(h5 && xtype && type_info);
734 if (!native_type_constant[1])
736 native_type_constant[1] = H5T_NATIVE_SCHAR;
737 native_type_constant[2] = H5T_NATIVE_SHORT;
738 native_type_constant[3] = H5T_NATIVE_INT;
739 native_type_constant[4] = H5T_NATIVE_FLOAT;
740 native_type_constant[5] = H5T_NATIVE_DOUBLE;
741 native_type_constant[6] = H5T_NATIVE_UCHAR;
742 native_type_constant[7] = H5T_NATIVE_USHORT;
743 native_type_constant[8] = H5T_NATIVE_UINT;
744 native_type_constant[9] = H5T_NATIVE_LLONG;
745 native_type_constant[10] = H5T_NATIVE_ULLONG;
749 if ((hdf_typeid = H5Dget_type(datasetid)) < 0)
754 if ((native_typeid = H5Tget_native_type(hdf_typeid, H5T_DIR_DEFAULT)) < 0)
758 if ((
class = H5Tget_class(native_typeid)) < 0)
762 if (
class == H5T_STRING ||
class == H5T_INTEGER ||
class == H5T_FLOAT)
765 if (!(*type_info = calloc(1,
sizeof(NC_TYPE_INFO_T))))
767 (*type_info)->class =
class;
771 if (
class == H5T_STRING)
773 if ((is_str = H5Tis_variable_str(native_typeid)) < 0)
776 if (is_str || H5Tget_size(hdf_typeid) > 1)
781 else if (
class == H5T_INTEGER ||
class == H5T_FLOAT)
783 for (t = 1; t < NUM_TYPES - 1; t++)
785 if ((equal = H5Tequal(native_typeid, native_type_constant[t])) < 0)
790 my_nc_type = nc_type_constant[t];
797 if (
class == H5T_INTEGER)
799 if ((order = H5Tget_order(hdf_typeid)) < 0)
801 if (order == H5T_ORDER_LE)
803 else if (order == H5T_ORDER_BE)
808 (*type_info)->endianness = endianness;
811 *xtype = nc_type_constant[t];
812 (*type_info)->nc_typeid = nc_type_constant[t];
813 (*type_info)->size = type_size[t];
814 if (!((*type_info)->name = malloc((strlen(nc_type_name[t]) + 1) *
sizeof(
char))))
816 strcpy((*type_info)->name, nc_type_name[t]);
817 (*type_info)->class =
class;
818 (*type_info)->hdf_typeid = hdf_typeid;
819 (*type_info)->native_typeid = native_typeid;
820 (*type_info)->close_hdf_typeid = 1;
826 if((type = nc4_rec_find_hdf_type(h5->root_grp, native_typeid)))
828 *xtype = type->nc_typeid;
835 if (H5Tclose(native_typeid) < 0)
837 if (H5Tclose(hdf_typeid) < 0)
850 read_hdf5_att(NC_GRP_INFO_T *grp, hid_t attid, NC_ATT_INFO_T *att)
852 hid_t spaceid = 0, file_typeid = 0;
857 hssize_t att_npoints;
858 H5T_class_t att_class;
859 int fixed_len_string = 0;
860 size_t fixed_size = 0;
863 LOG((5,
"read_hdf5_att: att->attnum %d att->name %s "
864 "att->xtype %d att->len %d", att->attnum, att->name,
865 att->xtype, att->len));
868 if ((file_typeid = H5Aget_type(attid)) < 0)
870 if ((att->native_typeid = H5Tget_native_type(file_typeid, H5T_DIR_DEFAULT)) < 0)
872 if ((att_class = H5Tget_class(att->native_typeid)) < 0)
874 if (att_class == H5T_STRING && !H5Tis_variable_str(att->native_typeid))
877 if (!(fixed_size = H5Tget_size(att->native_typeid)))
880 if ((retval = get_netcdf_type(grp->file->nc4_info, att->native_typeid, &(att->xtype))))
885 if ((spaceid = H5Aget_space(attid)) < 0)
890 if ((att_ndims = H5Sget_simple_extent_ndims(spaceid)) < 0)
892 if ((att_npoints = H5Sget_simple_extent_npoints(spaceid)) < 0)
897 if (att_ndims == 0 && att_npoints == 0)
902 dims[0] = att_npoints;
904 else if (att->xtype ==
NC_CHAR)
910 if (!(dims[0] = H5Tget_size(file_typeid)))
917 dims[0] = att_npoints;
927 if (H5Sget_simple_extent_dims(spaceid, dims, NULL) < 0)
938 if ((retval = nc4_get_typelen_mem(grp->file->nc4_info, att->xtype, 0,
941 if (att_class == H5T_VLEN)
943 if (!(att->vldata = malloc((
unsigned int)(att->len *
sizeof(hvl_t)))))
945 if (H5Aread(attid, att->native_typeid, att->vldata) < 0)
950 if (!(att->stdata = calloc(att->len,
sizeof(
char *))))
963 if (fixed_len_string)
966 char *contig_buf, *cur;
969 if (!(contig_buf = malloc(att->len * fixed_size *
sizeof(
char))))
973 if (H5Aread(attid, att->native_typeid, contig_buf) < 0)
980 for (i = 0; i < att->len; i++)
982 if (!(att->stdata[i] = malloc(fixed_size)))
984 strncpy(att->stdata[i], cur, fixed_size);
994 if (H5Aread(attid, att->native_typeid, att->stdata) < 0)
1000 if (!(att->data = malloc((
unsigned int)(att->len * type_size))))
1002 if (H5Aread(attid, att->native_typeid, att->data) < 0)
1007 if (H5Tclose(file_typeid) < 0)
1009 if (H5Sclose(spaceid) < 0)
1018 if (H5Tclose(file_typeid) < 0)
1020 if (spaceid > 0 && H5Sclose(spaceid) < 0)
1032 read_type(NC_GRP_INFO_T *grp,
char *type_name)
1034 NC_TYPE_INFO_T *type;
1036 hid_t hdf_typeid, native_typeid = 0;
1038 hid_t member_hdf_typeid, base_hdf_typeid = 0;
1039 char *member_name = NULL;
1040 size_t type_size = 0, member_offset;
1048 assert(grp && type_name);
1053 LOG((4,
"read_type: type_name %s grp->name %s", type_name, grp->name));
1055 if ((hdf_typeid = H5Topen2(grp->hdf_grpid, type_name, H5P_DEFAULT)) < 0)
1059 if ((native_typeid = H5Tget_native_type(hdf_typeid, H5T_DIR_DEFAULT)) < 0)
1063 if (!(type_size = H5Tget_size(native_typeid)))
1065 LOG((5,
"type_size %d", type_size));
1068 if ((
class = H5Tget_class(hdf_typeid)) < 0)
1083 if ((ret = H5Tis_variable_str(hdf_typeid)) < 0)
1093 if (!(base_hdf_typeid = H5Tget_super(native_typeid)))
1097 if (!(type_size = H5Tget_size(base_hdf_typeid)))
1101 if ((retval = get_netcdf_type(grp->file->nc4_info, base_hdf_typeid,
1104 LOG((5,
"base_hdf_typeid 0x%x type_size %d base_nc_type %d",
1105 base_hdf_typeid, type_size, base_nc_type));
1111 if (!(type_size = H5Tget_size(hdf_typeid)))
1113 LOG((5,
"type_size %d", type_size));
1120 if (!(base_hdf_typeid = H5Tget_super(hdf_typeid)))
1123 if (!(type_size = H5Tget_size(base_hdf_typeid)))
1126 if ((retval = get_netcdf_type(grp->file->nc4_info, base_hdf_typeid,
1129 LOG((5,
"base_hdf_typeid 0x%x type_size %d base_nc_type %d",
1130 base_hdf_typeid, type_size, base_nc_type));
1133 LOG((0,
"unknown class"));
1138 if ((retval = nc4_type_list_add(&grp->type, &type)))
1143 type->nc_typeid = grp->file->nc4_info->next_typeid++;
1144 type->size = type_size;
1145 if (!(type->name = malloc((strlen(type_name) + 1) *
sizeof(
char))))
1147 strcpy(type->name, type_name);
1148 type->class = ud_type_type;
1149 type->base_nc_type = base_nc_type;
1151 type->hdf_typeid = hdf_typeid;
1152 type->native_typeid = native_typeid;
1157 if ((nmembers = H5Tget_nmembers(hdf_typeid)) < 0)
1159 LOG((5,
"compound type has %d members", nmembers));
1160 for (m = 0; m < nmembers; m++)
1162 H5T_class_t mem_class;
1163 hid_t member_native_typeid;
1170 if ((member_hdf_typeid = H5Tget_member_type(type->native_typeid, m)) < 0)
1172 if ((member_native_typeid = H5Tget_native_type(member_hdf_typeid, H5T_DIR_DEFAULT)) < 0)
1176 member_name = H5Tget_member_name(type->native_typeid, m);
1177 if (!member_name || strlen(member_name) >
NC_MAX_NAME)
1181 member_offset = H5Tget_member_offset(type->native_typeid, m);
1184 if ((mem_class = H5Tget_class(member_hdf_typeid)) < 0)
1186 if (mem_class == H5T_ARRAY)
1188 if ((ndims = H5Tget_array_ndims(member_hdf_typeid)) < 0)
1190 if (H5Tget_array_dims(member_hdf_typeid, dims, NULL) != ndims)
1192 for (d = 0; d < ndims; d++)
1193 dim_size[d] = dims[d];
1195 if ((retval = get_netcdf_type(grp->file->nc4_info, H5Tget_super(member_hdf_typeid),
1202 if ((retval = get_netcdf_type(grp->file->nc4_info, member_native_typeid,
1210 if ((retval = nc4_field_list_add(&type->field, type->num_fields++, member_name,
1211 member_offset, H5Tget_super(member_hdf_typeid),
1212 H5Tget_super(member_native_typeid),
1213 member_xtype, ndims, dim_size)))
1218 if ((retval = nc4_field_list_add(&type->field, type->num_fields++, member_name,
1219 member_offset, member_hdf_typeid, member_native_typeid,
1220 member_xtype, 0, NULL)))
1228 else if (ud_type_type ==
NC_VLEN)
1230 type->base_hdf_typeid = base_hdf_typeid;
1232 else if (ud_type_type ==
NC_ENUM)
1235 type->base_hdf_typeid = base_hdf_typeid;
1238 if ((type->num_enum_members = H5Tget_nmembers(hdf_typeid)) < 0)
1242 if (!(value = calloc(1, type_size)))
1246 for (i = 0; i < type->num_enum_members; i++)
1249 if (!(member_name = H5Tget_member_name(hdf_typeid, i)))
1251 if (!member_name || strlen(member_name) >
NC_MAX_NAME)
1253 if (H5Tget_member_value(hdf_typeid, i, value) < 0)
1257 if ((retval = nc4_enum_member_add(&type->enum_member, type->size,
1258 member_name, value)))
1276 read_var(NC_GRP_INFO_T *grp, hid_t datasetid,
char *obj_name,
1277 size_t ndims,
int is_scale,
int num_scales, hid_t access_pid)
1284 char att_name[NC_MAX_HDF5_NAME + 1];
1286 #define CD_NELEMS_ZLIB 1
1287 #define CD_NELEMS_SZIP 4
1288 H5Z_filter_t filter;
1290 unsigned int cd_values[CD_NELEMS_SZIP];
1291 size_t cd_nelems = CD_NELEMS_SZIP;
1293 H5D_fill_value_t fill_status;
1294 H5D_layout_t layout;
1300 assert(obj_name && grp);
1301 LOG((4,
"read_var: obj_name %s", obj_name));
1304 if ((retval = nc4_var_list_add(&grp->var, &var)))
1308 var->hdf_datasetid = datasetid;
1309 var->varid = grp->nvars++;
1317 if (!(var->dim = calloc(var->ndims,
sizeof(NC_DIM_INFO_T *))))
1319 if (!(var->dimids = calloc(var->ndims,
sizeof(
int))))
1324 if ((H5Pget_chunk_cache(access_pid, &(var->chunk_cache_nelems),
1325 &(var->chunk_cache_size), &rdcc_w0)) < 0)
1327 var->chunk_cache_preemption = rdcc_w0;
1330 if (!(var->name = malloc((strlen(obj_name) + 1) *
sizeof(
char))))
1337 !strncmp(obj_name, NON_COORD_PREPEND, strlen(NON_COORD_PREPEND)))
1341 strcpy(var->name, &obj_name[strlen(NON_COORD_PREPEND)]);
1344 strcpy(var->name, obj_name);
1349 if ((propid = H5Dget_create_plist(datasetid)) < 0)
1356 if ((layout = H5Pget_layout(propid)) < -1)
1358 if (layout == H5D_CHUNKED)
1362 if (!(var->chunksizes = malloc(var->ndims *
sizeof(
size_t))))
1364 for (d = 0; d < var->ndims; d++)
1365 var->chunksizes[d] = chunksize[d];
1367 else if (layout == H5D_CONTIGUOUS)
1372 if ((num_filters = H5Pget_nfilters(propid)) < 0)
1374 for (f = 0; f < num_filters; f++)
1376 if ((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems,
1377 cd_values, 0, NULL, NULL)) < 0)
1381 case H5Z_FILTER_SHUFFLE:
1384 case H5Z_FILTER_FLETCHER32:
1385 var->fletcher32 = 1;
1387 case H5Z_FILTER_DEFLATE:
1389 if (cd_nelems != CD_NELEMS_ZLIB ||
1390 cd_values[0] > MAX_DEFLATE_LEVEL)
1392 var->deflate_level = cd_values[0];
1394 case H5Z_FILTER_SZIP:
1396 if (cd_nelems != CD_NELEMS_SZIP)
1398 var->options_mask = cd_values[0];
1399 var->pixels_per_block = cd_values[1];
1402 LOG((1,
"Yikes! Unknown filter type found on dataset!"));
1408 if ((retval = get_type_info2(grp->file->nc4_info, datasetid,
1409 &var->xtype, &var->type_info)))
1413 if (H5Pfill_value_defined(propid, &fill_status) < 0)
1417 if (fill_status == H5D_FILL_VALUE_USER_DEFINED)
1420 if (!var->fill_value)
1422 if (var->type_info->class ==
NC_VLEN)
1424 if (!(var->fill_value = malloc(
sizeof(
nc_vlen_t))))
1427 else if (var->type_info->size)
1429 if (!(var->fill_value = malloc(var->type_info->size)))
1434 if (!(var->fill_value = malloc(
sizeof(
char *))))
1440 if (H5Pget_fill_value(propid, var->type_info->native_typeid,
1441 var->fill_value) < 0)
1456 if ((retval = read_coord_dimids(var)))
1461 var->dimids[0] = grp->dim->dimid;
1462 var->dim[0] = grp->dim;
1466 if (num_scales && ndims &&
1467 !(var->dimscale_attached = calloc(ndims,
sizeof(
int))))
1473 if (!is_scale && num_scales)
1477 if (!(var->dimscale_hdf5_objids = malloc(ndims *
sizeof(
struct hdf5_objid))))
1479 for (d = 0; d < var->ndims; d++)
1481 LOG((5,
"read_var: about to iterate over scales for dim %d", d));
1482 if (H5DSiterate_scales(var->hdf_datasetid, d, NULL, dimscale_visitor,
1483 &(var->dimscale_hdf5_objids[d])) < 0)
1491 var->dimscale_attached[d]++;
1497 if ((natts = H5Aget_num_attrs(var->hdf_datasetid)) < 0)
1499 for (a = 0; a < natts; a++)
1504 if (attid && H5Aclose(attid) < 0)
1508 if ((attid = H5Aopen_idx(var->hdf_datasetid, (
unsigned int)a)) < 0)
1510 if (H5Aget_name(attid, NC_MAX_HDF5_NAME, att_name) < 0)
1512 LOG((4,
"read_var: a %d att_name %s", a, att_name));
1515 if (strcmp(att_name, REFERENCE_LIST) &&
1516 strcmp(att_name, CLASS) &&
1517 strcmp(att_name, DIMENSION_LIST) &&
1518 strcmp(att_name, NAME) &&
1519 strcmp(att_name, COORDINATES))
1523 if (!strcmp(att_name, NC_DIMID_ATT_NAME))
1530 if ((retval = nc4_att_list_add(&var->att)))
1532 for (att = var->att; att->next; att = att->next)
1536 att->attnum = var->natts++;
1537 if (!(att->name = malloc((strlen(att_name) + 1) *
sizeof(
char))))
1539 strcpy(att->name, att_name);
1543 if ((retval = read_hdf5_att(grp, attid, att)))
1553 if ((retval = nc4_adjust_var_cache(grp, var)))
1557 if (propid > 0 && H5Pclose(propid) < 0)
1562 if (attid > 0 && H5Aclose(attid) < 0)
1570 read_grp_atts(NC_GRP_INFO_T *grp)
1575 NC_TYPE_INFO_T *type;
1576 char obj_name[NC_MAX_HDF5_NAME + 1];
1580 num_obj = H5Aget_num_attrs(grp->hdf_grpid);
1581 for (i = 0; i < num_obj; i++)
1585 if ((attid = H5Aopen_idx(grp->hdf_grpid, (
unsigned int)i)) < 0)
1587 if (H5Aget_name(attid,
NC_MAX_NAME + 1, obj_name) < 0)
1589 LOG((3,
"reading attribute of _netCDF group, named %s", obj_name));
1595 if (!strcmp(obj_name, NC3_STRICT_ATT_NAME))
1600 if ((retval = nc4_att_list_add(&grp->att)))
1602 for (att = grp->att; att->next; att = att->next)
1607 if (!(att->name = malloc((max_len + 1) *
sizeof(
char))))
1609 strncpy(att->name, obj_name, max_len);
1610 att->name[max_len] = 0;
1611 att->attnum = grp->natts++;
1612 if ((retval = read_hdf5_att(grp, attid, att)))
1615 if ((retval = nc4_find_type(grp->file->nc4_info, att->xtype, &type)))
1618 att->class = type->class;
1623 if (attid > 0 && H5Aclose(attid) < 0)
1631 read_dataset(NC_GRP_INFO_T *grp,
char *obj_name)
1633 hid_t datasetid = 0;
1634 hid_t spaceid = 0, access_pid = 0;
1638 int dim_without_var = 0;
1643 if ((datasetid = H5Dopen2(grp->hdf_grpid, obj_name, H5P_DEFAULT)) < 0)
1647 if ((access_pid = H5Dget_access_plist(datasetid)) < 0)
1654 if ((spaceid = H5Dget_space(datasetid)) < 0)
1659 if ((ndims = H5Sget_simple_extent_ndims(spaceid)) < 0)
1663 if (H5Sget_simple_extent_dims(spaceid, dims, max_dims) < 0)
1667 if ((is_scale = H5DSis_scale(datasetid)) < 0)
1672 if ((retval = read_scale(grp, datasetid, obj_name, dims[0],
1673 max_dims[0], &dim_without_var)))
1681 num_scales = H5DSget_num_scales(datasetid, 0);
1689 if (!dim_without_var)
1690 if ((retval = read_var(grp, datasetid, obj_name, ndims,
1691 is_scale, num_scales, access_pid)))
1694 if (access_pid && H5Pclose(access_pid) < 0)
1699 if (spaceid && H5Sclose(spaceid) < 0)
1707 if (access_pid && H5Pclose(access_pid) < 0)
1712 if (datasetid && H5Dclose(datasetid) < 0)
1714 if (spaceid && H5Sclose(spaceid) <0)
1772 #define USE_ITERATE_CODE
1773 #ifdef USE_ITERATE_CODE
1776 nc4_rec_read_types_cb(hid_t grpid,
const char *name,
const H5L_info_t *info,
1780 H5I_type_t otype=-1;
1782 NC_GRP_INFO_T *child_grp;
1783 NC_GRP_INFO_T *grp = (NC_GRP_INFO_T *) (_op_data);
1784 NC_HDF5_FILE_INFO_T *h5 = grp->file->nc4_info;
1787 if ((oid = H5Oopen(grpid, name, H5P_DEFAULT)) < 0)
1788 return H5_ITER_ERROR;
1790 if ((otype = H5Iget_type( oid ))<0) {
1792 return H5_ITER_ERROR;
1799 if (otype == H5I_GROUP)
1801 LOG((3,
"found group %s", oname));
1802 if (nc4_grp_list_add(&(grp->children), h5->next_nc_grpid++,
1803 grp, grp->file, oname, &child_grp))
1804 return H5_ITER_ERROR;
1806 if (nc4_rec_read_types(child_grp))
1807 return H5_ITER_ERROR;
1809 else if (otype == H5I_DATATYPE)
1811 LOG((3,
"found datatype %s", oname));
1812 if (read_type(grp, oname))
1813 return H5_ITER_ERROR;
1816 return (H5_ITER_CONT);
1820 nc4_rec_read_types(NC_GRP_INFO_T *grp)
1825 unsigned crt_order_flags = 0;
1826 NC_HDF5_FILE_INFO_T *h5 = grp->file->nc4_info;
1828 assert(grp && grp->name);
1829 LOG((3,
"nc4_rec_read_types: grp->name %s", grp->name));
1833 if (!grp->hdf_grpid)
1837 if ((grp->hdf_grpid = H5Gopen2(grp->parent->hdf_grpid,
1838 grp->name, H5P_DEFAULT)) < 0)
1843 if ((grp->hdf_grpid = H5Gopen2(grp->file->nc4_info->hdfid,
1844 "/", H5P_DEFAULT)) < 0)
1848 assert(grp->hdf_grpid > 0);
1850 pid = H5Gget_create_plist(grp->hdf_grpid);
1851 H5Pget_link_creation_order(pid, &crt_order_flags);
1854 crt_order_flags = crt_order_flags & H5_INDEX_CRT_ORDER;
1856 if (crt_order_flags == H5_INDEX_CRT_ORDER)
1858 res = H5Literate(grp->hdf_grpid, H5_INDEX_CRT_ORDER, H5_ITER_INC,
1859 &idx, nc4_rec_read_types_cb, (
void *)grp);
1863 if (!idx && !h5->no_write)
1866 res = H5Literate(grp->hdf_grpid, H5_INDEX_NAME, H5_ITER_INC,
1867 &idx, nc4_rec_read_types_cb, (
void *)grp);
1875 nc4_rec_read_vars_cb(hid_t grpid,
const char *name,
const H5L_info_t *info,
1879 H5I_type_t otype=-1;
1881 NC_GRP_INFO_T *child_grp;
1882 NC_GRP_INFO_T *grp = (NC_GRP_INFO_T *) (_op_data);
1884 NC_HDF5_FILE_INFO_T *h5 = grp->file->nc4_info;
1889 if ((oid = H5Oopen(grpid, name, H5P_DEFAULT)) < 0)
1890 return H5_ITER_ERROR;
1892 if ((otype = H5Iget_type( oid ))<0) {
1894 return H5_ITER_ERROR;
1904 LOG((3,
"re-encountering group %s", oname));
1907 for (child_grp = grp->children; child_grp; child_grp = child_grp->next)
1908 if (!strcmp(child_grp->name, oname))
1911 return H5_ITER_ERROR;
1914 if (nc4_rec_read_vars(child_grp))
1915 return H5_ITER_ERROR;
1918 LOG((3,
"found dataset %s", oname));
1922 if (read_dataset(grp, oname))
1923 return H5_ITER_ERROR;
1926 LOG((3,
"already handled type %s", oname));
1929 LOG((0,
"Unknown object class %d in nc4_rec_read_vars!", otype));
1931 return (H5_ITER_CONT);
1935 nc4_rec_read_vars(NC_GRP_INFO_T *grp)
1941 unsigned crt_order_flags = 0;
1942 NC_HDF5_FILE_INFO_T *h5 = grp->file->nc4_info;
1944 assert(grp && grp->name && grp->hdf_grpid > 0);
1945 LOG((3,
"nc4_rec_read_vars: grp->name %s", grp->name));
1947 pid = H5Gget_create_plist(grp->hdf_grpid);
1948 H5Pget_link_creation_order(pid, &crt_order_flags);
1951 crt_order_flags = crt_order_flags & H5_INDEX_CRT_ORDER;
1953 if (crt_order_flags == H5_INDEX_CRT_ORDER)
1955 res = H5Literate(grp->hdf_grpid, H5_INDEX_CRT_ORDER, H5_ITER_INC,
1956 &idx, nc4_rec_read_vars_cb, (
void *)grp);
1960 if (!idx && !h5->no_write)
1963 res = H5Literate(grp->hdf_grpid, H5_INDEX_NAME, H5_ITER_INC,
1964 &idx, nc4_rec_read_vars_cb, (
void *)grp);
1970 if ((retval = read_grp_atts(grp)))
1982 struct nc_hdf5_link_info
1985 H5I_type_t obj_type;
2007 visit_link(hid_t g_id,
const char *name,
const H5L_info_t *info,
2018 strncpy(((
struct nc_hdf5_link_info *)op_data)->name, name,
2022 if ((
id = H5Oopen_by_addr(g_id, info->u.address)) < 0)
2026 if ((((
struct nc_hdf5_link_info *)op_data)->obj_type = H5Iget_type(
id)) < 0)
2030 if (H5Oclose(
id) < 0)
2041 nc4_iterate_link(
int *ordering_checked,
int *creation_ordering,
2042 hid_t grpid, hsize_t *idx,
struct nc_hdf5_link_info *link_info)
2046 if (*creation_ordering)
2051 res = H5Literate(grpid, H5_INDEX_CRT_ORDER, H5_ITER_INC,
2052 idx, visit_link, (
void *)link_info);
2053 if (res < 0 && *ordering_checked)
2058 if (!*creation_ordering || res < 0)
2060 if (H5Literate(grpid, H5_INDEX_NAME, H5_ITER_INC, idx,
2061 visit_link, link_info) != 1)
2065 *creation_ordering = 0;
2068 *ordering_checked = 1;
2074 nc4_rec_read_types(NC_GRP_INFO_T *grp)
2077 NC_HDF5_FILE_INFO_T *h5 = grp->file->nc4_info;
2078 NC_GRP_INFO_T *child_grp;
2080 struct nc_hdf5_link_info link_info;
2081 int ordering_checked = 0;
2082 int creation_ordering = 1;
2085 assert(grp && grp->name);
2086 LOG((3,
"nc4_rec_read_types: grp->name %s", grp->name));
2090 if (!grp->hdf_grpid)
2094 if ((grp->hdf_grpid = H5Gopen2(grp->parent->hdf_grpid,
2095 grp->name, H5P_DEFAULT)) < 0)
2100 if ((grp->hdf_grpid = H5Gopen2(grp->file->nc4_info->hdfid,
2101 "/", H5P_DEFAULT)) < 0)
2105 assert(grp->hdf_grpid > 0);
2108 if (H5Gget_num_objs(grp->hdf_grpid, &num_obj) < 0)
2112 for (i = 0; i < num_obj; i++)
2114 if ((retval = nc4_iterate_link(&ordering_checked, &creation_ordering,
2115 grp->hdf_grpid, &idx, &link_info)))
2119 if (!i && !creation_ordering && !h5->no_write)
2123 if (link_info.obj_type == H5I_GROUP)
2125 LOG((3,
"found group %s", link_info.name));
2126 if ((retval = nc4_grp_list_add(&(grp->children), h5->next_nc_grpid++,
2127 grp, grp->file, link_info.name, &child_grp)))
2129 if ((retval = nc4_rec_read_types(child_grp)))
2132 else if (link_info.obj_type == H5I_DATATYPE)
2134 LOG((3,
"found datatype %s", link_info.name));
2135 if ((retval = read_type(grp, link_info.name)))
2147 nc4_rec_read_vars(NC_GRP_INFO_T *grp)
2150 NC_GRP_INFO_T *child_grp;
2151 struct nc_hdf5_link_info link_info;
2153 int ordering_checked = 0;
2154 int creation_ordering = 1;
2157 assert(grp && grp->name && grp->hdf_grpid > 0);
2158 LOG((3,
"nc4_rec_read_vars: grp->name %s", grp->name));
2161 if (H5Gget_num_objs(grp->hdf_grpid, &num_obj) < 0)
2165 for (i = 0; i < num_obj; i++)
2167 if ((retval = nc4_iterate_link(&ordering_checked, &creation_ordering,
2168 grp->hdf_grpid, &idx, &link_info)))
2172 switch(link_info.obj_type)
2175 LOG((3,
"re-encountering group %s", link_info.name));
2178 for (child_grp = grp->children; child_grp; child_grp = child_grp->next)
2179 if (!strcmp(child_grp->name, link_info.name))
2185 if ((retval = nc4_rec_read_vars(child_grp)))
2189 LOG((3,
"found dataset %s", link_info.name));
2193 if ((retval = read_dataset(grp, link_info.name)))
2197 LOG((3,
"already handled type %s", link_info.name));
2200 LOG((0,
"Unknown object class %d in nc4_rec_read_vars!",
2201 link_info.obj_type));
2206 if ((retval = read_grp_atts(grp)))
2217 nc4_open_file(
const char *path,
int mode, MPI_Comm comm,
2218 MPI_Info info, NC_FILE_INFO_T *nc)
2220 hid_t fapl_id = H5P_DEFAULT;
2221 unsigned flags = (mode &
NC_WRITE) ?
2222 H5F_ACC_RDWR : H5F_ACC_RDONLY;
2225 LOG((3,
"nc4_open_file: path %s mode %d", path, mode));
2229 if(mode & NC_DISKLESS)
2233 if ((retval = nc4_nc4f_list_add(nc, path, mode)))
2235 assert(nc->nc4_info && nc->nc4_info->root_grp);
2240 if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
2246 if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_SEMI))
2249 if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_STRONG))
2256 if (mode & NC_MPIIO || mode & NC_MPIPOSIX)
2258 nc->nc4_info->parallel++;
2259 if (mode & NC_MPIIO)
2261 LOG((4,
"opening parallel file with MPI/IO"));
2262 if (H5Pset_fapl_mpio(fapl_id, comm, info) < 0)
2267 LOG((4,
"opening parallel file with MPI/posix"));
2268 if (H5Pset_fapl_mpiposix(fapl_id, comm, 0) < 0)
2273 if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size,
2274 nc4_chunk_cache_preemption) < 0)
2276 LOG((4,
"nc4_open_file: set HDF raw chunk cache to size %d nelems %d preemption %f",
2277 nc4_chunk_cache_size, nc4_chunk_cache_nelems, nc4_chunk_cache_preemption));
2283 if ((nc->nc4_info->hdfid = H5Fopen(path, flags, fapl_id)) < 0)
2288 nc->nc4_info->no_write++;
2294 if ((retval = nc4_rec_read_types(nc->nc4_info->root_grp)))
2296 if ((retval = nc4_rec_read_vars(nc->nc4_info->root_grp)))
2301 if ((retval = nc4_rec_match_dimscales(nc->nc4_info->root_grp)))
2307 log_metadata_nc(nc);
2311 if (H5Pclose(fapl_id) < 0)
2320 if (fapl_id != H5P_DEFAULT) H5Pclose(fapl_id);
2324 if (nc->nc4_info->hdfid > 0) H5Fclose(nc->nc4_info->hdfid);
2325 if (nc->nc4_info) free(nc->nc4_info);
2332 get_netcdf_type_from_hdf4(NC_HDF5_FILE_INFO_T *h5, int32 hdf4_typeid,
2333 nc_type *xtype, NC_TYPE_INFO_T *type_info)
2336 assert(h5 && xtype);
2384 if (hdf4_typeid == DFNT_FLOAT32 || hdf4_typeid == DFNT_FLOAT64)
2385 type_info->class = H5T_FLOAT;
2386 else if (hdf4_typeid == DFNT_CHAR)
2387 type_info->class = H5T_STRING;
2389 type_info->class = H5T_INTEGER;
2391 type_info->nc_typeid = *xtype;
2392 if (type_info->name)
2393 free(type_info->name);
2394 if (!(type_info->name = malloc((strlen(nc_type_name[t]) + 1) *
sizeof(
char))))
2396 strcpy(type_info->name, nc_type_name[t]);
2406 nc4_open_hdf4_file(
const char *path,
int mode, NC_FILE_INFO_T *nc)
2409 NC_HDF5_FILE_INFO_T *h5;
2413 int32 num_datasets, num_gatts;
2418 LOG((3,
"nc4_open_hdf4_file: path %s mode %d", path, mode));
2422 if (mode & NC_WRITE)
2426 if ((retval = nc4_nc4f_list_add(nc, path, mode)))
2428 assert(nc->nc4_info && nc->nc4_info->root_grp);
2435 if ((h5->sdid = SDstart(path, DFACC_READ)) == FAIL)
2439 if (SDfileinfo(h5->sdid, &num_datasets, &num_gatts))
2443 for (a = 0; a < num_gatts; a++)
2445 int32 att_data_type, att_count;
2446 size_t att_type_size;
2449 if ((retval = nc4_att_list_add(&h5->root_grp->att)))
2451 for (att = h5->root_grp->att; att->next; att = att->next)
2453 att->attnum = grp->natts++;
2459 if (SDattrinfo(h5->sdid, a, att->name, &att_data_type, &att_count))
2461 if ((retval = get_netcdf_type_from_hdf4(h5, att_data_type,
2462 &att->xtype, NULL)))
2464 att->len = att_count;
2467 if ((retval = nc4_get_typelen_mem(h5, att->xtype, 0, &att_type_size)))
2469 if (!(att->data = malloc(att_type_size * att->len)))
2473 if (SDreadattr(h5->sdid, a, att->data))
2478 for (v = 0; v < num_datasets; v++)
2480 int32 data_type, num_atts;
2482 size_t var_type_size;
2486 if ((retval = nc4_var_list_add(&grp->var, &var)))
2488 var->varid = grp->nvars++;
2490 var->written_to = 1;
2493 if ((var->sdsid = SDselect(h5->sdid, v)) == FAIL)
2499 if (SDgetinfo(var->sdsid, var->name, &rank, dimsize, &data_type, &num_atts))
2502 var->hdf4_data_type = data_type;
2505 if (!(var->type_info = calloc(1,
sizeof(NC_TYPE_INFO_T))))
2507 if ((retval = get_netcdf_type_from_hdf4(h5, data_type, &var->xtype, var->type_info)))
2509 if ((retval = nc4_get_typelen_mem(h5, var->xtype, 0, &var_type_size)))
2511 var->type_info->size = var_type_size;
2512 LOG((3,
"reading HDF4 dataset %s, rank %d netCDF type %d", var->name,
2516 if (!(var->fill_value = malloc(var_type_size)))
2518 if (SDgetfillvalue(var->sdsid, var->fill_value))
2521 free(var->fill_value);
2522 var->fill_value = NULL;
2528 if (!(var->dim = malloc(
sizeof(NC_DIM_INFO_T *) * var->ndims)))
2530 if (!(var->dimids = malloc(
sizeof(
int) * var->ndims)))
2535 for (d = 0; d < var->ndims; d++)
2537 int32 dimid, dim_len, dim_data_type, dim_num_attrs;
2541 if ((dimid = SDgetdimid(var->sdsid, d)) == FAIL)
2543 if (SDdiminfo(dimid, dim_name, &dim_len, &dim_data_type,
2549 for (dim = grp->dim; dim; dim = dim->next)
2550 if (!strcmp(dim->name, dim_name))
2556 LOG((4,
"adding dimension %s for HDF4 dataset %s",
2557 dim_name, var->name));
2558 if ((retval = nc4_dim_list_add(&grp->dim)))
2562 dim->dimid = grp->file->nc4_info->next_dimid++;
2567 strcpy(dim->name, dim_name);
2571 dim->len = *dimsize;
2575 var->dimids[d] = dim->dimid;
2579 for (a = 0; a < num_atts; a++)
2581 int32 att_data_type, att_count;
2582 size_t att_type_size;
2585 if ((retval = nc4_att_list_add(&var->att)))
2587 for (att = var->att; att->next; att = att->next)
2589 att->attnum = var->natts++;
2595 if (SDattrinfo(var->sdsid, a, att->name, &att_data_type, &att_count))
2597 if ((retval = get_netcdf_type_from_hdf4(h5, att_data_type,
2598 &att->xtype, NULL)))
2600 att->len = att_count;
2603 if ((retval = nc4_get_typelen_mem(h5, att->xtype, 0, &att_type_size)))
2605 if (!(att->data = malloc(att_type_size * att->len)))
2609 if (SDreadattr(var->sdsid, a, att->data))
2617 log_metadata_nc(h5->root_grp->file);
2625 NC4_open(
const char *path,
int mode,
int basepe,
size_t *chunksizehintp,
2626 int use_parallel,
void *mpidata, NC_Dispatch *dispatch, NC **ncpp)
2629 NC_FILE_INFO_T *nc_file;
2634 int comm = 0, info = 0;
2638 assert(ncpp && path);
2640 LOG((1,
"nc_open_file: path %s mode %d comm %d info %d",
2641 path, mode, comm, info));
2646 NC_MPI_INFO *nmi = (NC_MPI_INFO *)mpidata;
2647 comm = nmi->comm; info = nmi->info;
2654 if (H5Eset_auto(NULL, NULL) < 0)
2655 LOG((0,
"Couldn't turn off HDF5 error messages!"));
2656 LOG((1,
"HDF5 error messages turned off!"));
2663 if (mode & ~(NC_WRITE |
NC_SHARE | NC_MPIIO | NC_MPIPOSIX |
2665 (mode & NC_MPIIO && mode & NC_MPIPOSIX))
2669 if ((res = nc_check_for_hdf(path, use_parallel, comm, info, &hdf_file)))
2674 if ((res = nc4_file_list_add(&nc_file,dispatch)))
2678 if (hdf_file == NC_HDF5_FILE)
2680 nc_file->int_ncid = nc_file->ext_ncid;
2681 res = nc4_open_file(path, mode, comm, info, nc_file);
2683 else if (hdf_file == NC_HDF4_FILE)
2685 nc_file->int_ncid = nc_file->ext_ncid;
2686 res = nc4_open_hdf4_file(path, mode, nc_file);
2691 int pnetcdf_nvars, i;
2693 res = ncmpi_open(comm, path, mode, info, &(nc_file->int_ncid));
2694 nc_file->pnetcdf_file++;
2698 res = ncmpi_begin_indep_data(nc_file->int_ncid);
2704 res = ncmpi_inq_nvars(nc_file->int_ncid, &pnetcdf_nvars);
2705 for (i = 0; i < pnetcdf_nvars; i++)
2706 res = ncmpi_inq_varndims(nc_file->int_ncid, i,
2707 &(nc_file->pnetcdf_ndims[i]));
2721 if(nc_file != NULL) nc4_file_list_del(nc_file);
2725 *ncpp = (NC*)nc_file;
2737 NC4_set_fill(
int ncid,
int fillmode,
int *old_modep)
2741 LOG((2,
"nc_set_fill: ncid 0x%x fillmode %d", ncid, fillmode));
2743 if (!(nc = nc4_find_nc_file(ncid)))
2747 assert(nc->nc4_info);
2750 if (nc->nc4_info->no_write)
2759 *old_modep = nc->nc4_info->fill_mode;
2761 nc->nc4_info->fill_mode = fillmode;
2773 LOG((1,
"nc_redef: ncid 0x%x", ncid));
2776 if (!(nc = nc4_find_nc_file(ncid)))
2781 if (nc->pnetcdf_file)
2782 return ncmpi_redef(nc->int_ncid);
2786 assert(nc->nc4_info);
2789 if (nc->nc4_info->flags & NC_INDEF)
2793 if (nc->nc4_info->no_write)
2797 nc->nc4_info->flags |= NC_INDEF;
2801 nc->nc4_info->redef++;
2809 NC4__enddef(
int ncid,
size_t h_minfree,
size_t v_align,
2810 size_t v_minfree,
size_t r_align)
2812 if (!nc4_find_nc_file(ncid))
2815 return NC4_enddef(ncid);
2820 static int NC4_enddef(
int ncid)
2824 LOG((1,
"nc_enddef: ncid 0x%x", ncid));
2826 if (!(nc = nc4_find_nc_file(ncid)))
2830 if (nc->pnetcdf_file)
2833 res = ncmpi_enddef(nc->int_ncid);
2836 if (nc->pnetcdf_access_mode == NC_INDEPENDENT)
2837 res = ncmpi_begin_indep_data(nc->int_ncid);
2844 assert(nc->nc4_info);
2846 return nc4_enddef_netcdf4_file(nc->nc4_info);
2852 sync_netcdf4_file(NC_HDF5_FILE_INFO_T *h5)
2857 LOG((3,
"sync_netcdf4_file"));
2861 if (h5->flags & NC_INDEF)
2867 h5->flags ^= NC_INDEF;
2876 log_metadata_nc(h5->root_grp->file);
2882 if ((retval = nc4_rec_write_types(h5->root_grp)))
2884 if ((retval = nc4_rec_write_metadata(h5->root_grp)))
2888 H5Fflush(h5->hdfid, H5F_SCOPE_GLOBAL);
2905 LOG((2,
"nc_sync: ncid 0x%x", ncid));
2907 if (!(nc = nc4_find_nc_file(ncid)))
2912 if (nc->pnetcdf_file)
2913 return ncmpi_sync(nc->int_ncid);
2917 assert(nc->nc4_info);
2920 if (nc->nc4_info && nc->nc4_info->flags & NC_INDEF)
2928 return sync_netcdf4_file(nc->nc4_info);
2935 close_netcdf4_file(NC_HDF5_FILE_INFO_T *h5,
int abort)
2939 assert(h5 && h5->root_grp);
2940 LOG((3,
"close_netcdf4_file: h5->path %s abort %d",
2944 if (h5->flags & NC_INDEF)
2945 h5->flags ^= NC_INDEF;
2949 if (!h5->no_write && !abort)
2950 if ((retval = sync_netcdf4_file(h5)))
2955 if ((retval = nc4_rec_grp_del(&h5->root_grp, h5->root_grp)))
2962 if (SDend(h5->sdid))
2968 if (H5Fclose(h5->hdfid) < 0)
2971 nobjs = H5Fget_obj_count(h5->hdfid, H5F_OBJ_ALL);
2975 }
else if(nobjs > 0) {
2981 LOG((0,
"There are %d HDF5 objects open!", nobjs));
3009 int delete_file = 0;
3013 LOG((2,
"nc_abort: ncid 0x%x", ncid));
3016 if (!(nc = nc4_find_nc_file(ncid)))
3021 if (nc->pnetcdf_file)
3022 return ncmpi_abort(nc->int_ncid);
3026 assert(nc->nc4_info);
3029 if (nc->nc4_info->flags & NC_INDEF && !nc->nc4_info->redef)
3032 strcpy(path, nc->nc4_info->path);
3038 if ((retval = close_netcdf4_file(nc->nc4_info, 1)))
3046 nc4_file_list_del(nc);
3057 NC_HDF5_FILE_INFO_T *h5;
3060 LOG((1,
"nc_close: ncid 0x%x", ncid));
3063 if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5)))
3068 if (nc->pnetcdf_file)
3069 return ncmpi_close(nc->int_ncid);
3079 if ((retval = close_netcdf4_file(grp->file->nc4_info, 0)))
3085 nc4_file_list_del(nc);
3088 if(count_NCList() == 0)
3089 nc4_file_list_free();
3097 NC4_inq(
int ncid,
int *ndimsp,
int *nvarsp,
int *nattsp,
int *unlimdimidp)
3100 NC_HDF5_FILE_INFO_T *h5;
3107 LOG((2,
"nc_inq: ncid 0x%x", ncid));
3110 if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5)))
3115 if (nc->pnetcdf_file)
3116 return ncmpi_inq(nc->int_ncid, ndimsp, nvarsp, nattsp, unlimdimidp);
3120 assert(h5 && grp && nc);
3126 for (dim = grp->dim; dim; dim = dim->next)
3132 for (var = grp->var; var; var= var->next)
3138 for (att = grp->att; att; att = att->next)
3152 for (dim = grp->dim; dim; dim = dim->next)
3155 *unlimdimidp = dim->dimid;
3166 nc4_enddef_netcdf4_file(NC_HDF5_FILE_INFO_T *h5)
3169 LOG((3,
"nc4_enddef_netcdf4_file"));
3172 if (!(h5->flags & NC_INDEF))
3176 h5->flags ^= NC_INDEF;
3181 return sync_netcdf4_file(h5);
3188 if (num_plists || num_spaces)
3197 nc_use_parallel_enabled()