netCDF  4.2.1.1
 All Data Structures Files Functions Variables Typedefs Macros Groups Pages
nccopy.c
1 /*********************************************************************
2  * Copyright 2010, University Corporation for Atmospheric Research
3  * See netcdf/README file for copying and redistribution conditions.
4  * Thanks to Philippe Poilbarbe and Antonio S. CofiƱo for
5  * compression additions.
6  * $Id: nccopy.c 400 2010-08-27 21:02:52Z russ $
7  *********************************************************************/
8 
9 #include "config.h" /* for USE_NETCDF4 macro */
10 #include <stdlib.h>
11 #ifdef HAVE_GETOPT_H
12 #include <getopt.h>
13 #endif
14 #ifndef _WIN32
15 #include <unistd.h>
16 #endif
17 #include <string.h>
18 #include <netcdf.h>
19 #include "nciter.h"
20 #include "chunkspec.h"
21 #include "utils.h"
22 #include "dimmap.h"
23 
24 /* default bytes of memory we are willing to allocate for variable
25  * values during copy */
26 #define COPY_BUFFER_SIZE (5000000)
27 #define COPY_CHUNKCACHE_PREEMPTION (1.0f) /* for copying, can eject fully read chunks */
28 #define SAME_AS_INPUT (-1) /* default, if kind not specified */
29 #define CHUNK_THRESHOLD (1024) /* variables with fewer bytes don't get chunked */
30 
31 #ifndef USE_NETCDF4
32 #define NC_CLASSIC_MODEL 0x0100 /* Enforce classic model if netCDF-4 not available. */
33 #endif
34 
35 /* Global variables for command-line requests */
36 char *progname; /* for error messages */
37 static int option_kind = SAME_AS_INPUT;
38 static int option_deflate_level = -1; /* default, compress output only if input compressed */
39 static int option_shuffle_vars = NC_NOSHUFFLE; /* default, no shuffling on compression */
40 static int option_fix_unlimdims = 0; /* default, preserve unlimited dimensions */
41 static char* option_chunkspec = 0; /* default, no chunk specification */
42 static size_t option_copy_buffer_size = COPY_BUFFER_SIZE;
43 static size_t option_chunk_cache_size = CHUNK_CACHE_SIZE; /* default from config.h */
44 static size_t option_chunk_cache_nelems = CHUNK_CACHE_NELEMS; /* default from config.h */
45 static int option_compute_chunkcaches = 0; /* default, don't try still flaky estimate of
46  * chunk cache for each variable */
47 static int option_read_diskless = 0; /* default, don't read input into memory on open */
48 static int option_write_diskless = 0; /* default, don't write output to diskless file */
49 
50 /* get group id in output corresponding to group igrp in input,
51  * given parent group id (or root group id) parid in output. */
52 static int
53 get_grpid(int igrp, int parid, int *ogrpp) {
54  int stat = NC_NOERR;
55  int ogid = parid; /* like igrp but in output file */
56 #ifdef USE_NETCDF4
57  int inparid;
58 
59  /* if not root group, get corresponding output groupid from group name */
60  stat = nc_inq_grp_parent(igrp, &inparid);
61  if(stat == NC_NOERR) { /* not root group */
62  char grpname[NC_MAX_NAME + 1];
63  NC_CHECK(nc_inq_grpname(igrp, grpname));
64  NC_CHECK(nc_inq_grp_ncid(parid, grpname, &ogid));
65  } else if(stat == NC_ENOGRP) { /* root group */
66  stat = NC_NOERR;
67  } else {
68  NC_CHECK(stat);
69  }
70 #endif /* USE_NETCDF4 */
71  *ogrpp = ogid;
72  return stat;
73 }
74 
75 
76 #ifdef USE_NETCDF4
77 /* Get parent id needed to define a new group from its full name in an
78  * open file identified by ncid. Assumes all intermediate groups are
79  * already defined. */
80 static int
81 nc_inq_parid(int ncid, const char *fullname, int *locidp) {
82  int stat = NC_NOERR;
83  char *parent = strdup(fullname);
84  char *slash = "/"; /* groupname separator */
85  char *last_slash;
86  if(parent == NULL) {
87  NC_CHECK(NC_ENOMEM);
88  } else
89  last_slash = strrchr(parent, '/');
90  if(last_slash == parent) { /* parent is root */
91  free(parent);
92  parent = strdup(slash);
93  } else {
94  *last_slash = '\0'; /* truncate to get parent name */
95  }
96  NC_CHECK(nc_inq_grp_full_ncid(ncid, parent, locidp));
97  free(parent);
98  return stat;
99 }
100 
101 /* Return size of chunk in bytes for a variable varid in a group igrp, or 0 if
102  * layout is contiguous */
103 static int
104 inq_var_chunksize(int igrp, int varid, size_t* chunksizep) {
105  int stat = NC_NOERR;
106  int ndims;
107  size_t *chunksizes;
108  int dim;
109  int contig = 1;
110  nc_type vartype;
111  size_t value_size;
112  size_t prod;
113 
114  NC_CHECK(nc_inq_vartype(igrp, varid, &vartype));
115  /* from type, get size in memory needed for each value */
116  NC_CHECK(nc_inq_type(igrp, vartype, NULL, &value_size));
117  prod = value_size;
118  NC_CHECK(nc_inq_varndims(igrp, varid, &ndims));
119  chunksizes = (size_t *) emalloc((ndims + 1) * sizeof(size_t));
120  if(ndims > 0) {
121  NC_CHECK(nc_inq_var_chunking(igrp, varid, &contig, NULL));
122  }
123  if(contig == 1) {
124  *chunksizep = 0;
125  } else {
126  NC_CHECK(nc_inq_var_chunking(igrp, varid, &contig, chunksizes));
127  for(dim = 0; dim < ndims; dim++) {
128  prod *= chunksizes[dim];
129  }
130  *chunksizep = prod;
131  }
132  free(chunksizes);
133  return stat;
134 }
135 
136 /* Return estimated number of elems required in chunk cache and
137  * estimated size of chunk cache adequate to efficiently copy input
138  * variable ivarid to output variable ovarid, which may have different
139  * chunk size and shape */
140 static int
141 inq_var_chunking_params(int igrp, int ivarid, int ogrp, int ovarid,
142  size_t* chunkcache_sizep,
143  size_t *chunkcache_nelemsp,
144  float * chunkcache_preemptionp)
145 {
146  int stat = NC_NOERR;
147  int ndims;
148  size_t *ichunksizes, *ochunksizes;
149  int dim;
150  int icontig = 1, ocontig = 1;
151  nc_type vartype;
152  size_t value_size;
153  size_t prod, iprod, oprod;
154  size_t nelems;
155  *chunkcache_nelemsp = CHUNK_CACHE_NELEMS;
156  *chunkcache_sizep = CHUNK_CACHE_SIZE;
157  *chunkcache_preemptionp = COPY_CHUNKCACHE_PREEMPTION;
158 
159  NC_CHECK(nc_inq_varndims(igrp, ivarid, &ndims));
160  if(ndims > 0) {
161  NC_CHECK(nc_inq_var_chunking(igrp, ivarid, &icontig, NULL));
162  NC_CHECK(nc_inq_var_chunking(ogrp, ovarid, &ocontig, NULL));
163  }
164  if(icontig == 1 && ocontig == 1) { /* no chunking in input or output */
165  *chunkcache_nelemsp = 0;
166  *chunkcache_sizep = 0;
167  *chunkcache_preemptionp = 0;
168  return stat;
169  }
170 
171  NC_CHECK(nc_inq_vartype(igrp, ivarid, &vartype));
172  NC_CHECK(nc_inq_type(igrp, vartype, NULL, &value_size));
173  iprod = value_size;
174 
175  if(icontig == 0 && ocontig == 1) { /* chunking only in input */
176  *chunkcache_nelemsp = 1; /* read one input chunk at a time */
177  *chunkcache_sizep = iprod;
178  *chunkcache_preemptionp = 1.0f;
179  return stat;
180  }
181 
182  ichunksizes = (size_t *) emalloc((ndims + 1) * sizeof(size_t));
183  if(icontig == 1) { /* if input contiguous, treat as if chunked on
184  * first dimension */
185  ichunksizes[0] = 1;
186  for(dim = 1; dim < ndims; dim++) {
187  ichunksizes[dim] = dim;
188  }
189  } else {
190  NC_CHECK(nc_inq_var_chunking(igrp, ivarid, &icontig, ichunksizes));
191  }
192 
193  /* now can assume chunking in both input and output */
194  ochunksizes = (size_t *) emalloc((ndims + 1) * sizeof(size_t));
195  NC_CHECK(nc_inq_var_chunking(ogrp, ovarid, &ocontig, ochunksizes));
196 
197  nelems = 1;
198  oprod = value_size;
199  for(dim = 0; dim < ndims; dim++) {
200  nelems += 1 + (ichunksizes[dim] - 1) / ochunksizes[dim];
201  iprod *= ichunksizes[dim];
202  oprod *= ochunksizes[dim];
203  }
204  prod = iprod + oprod * (nelems - 1);
205  *chunkcache_nelemsp = nelems;
206  *chunkcache_sizep = prod;
207  free(ichunksizes);
208  free(ochunksizes);
209  return stat;
210 }
211 
212 /* Forward declaration, because copy_type, copy_vlen_type call each other */
213 static int copy_type(int igrp, nc_type typeid, int ogrp);
214 
215 /*
216  * copy a user-defined variable length type in the group igrp to the
217  * group ogrp
218  */
219 static int
220 copy_vlen_type(int igrp, nc_type itype, int ogrp)
221 {
222  int stat = NC_NOERR;
223  nc_type ibasetype;
224  nc_type obasetype; /* base type in target group */
225  char name[NC_MAX_NAME];
226  size_t size;
227  char basename[NC_MAX_NAME];
228  size_t basesize;
229  nc_type vlen_type;
230 
231  NC_CHECK(nc_inq_vlen(igrp, itype, name, &size, &ibasetype));
232  /* to get base type id in target group, use name of base type in
233  * source group */
234  NC_CHECK(nc_inq_type(igrp, ibasetype, basename, &basesize));
235  stat = nc_inq_typeid(ogrp, basename, &obasetype);
236  /* if no such type, create it now */
237  if(stat == NC_EBADTYPE) {
238  NC_CHECK(copy_type(igrp, ibasetype, ogrp));
239  stat = nc_inq_typeid(ogrp, basename, &obasetype);
240  }
241  NC_CHECK(stat);
242 
243  /* Now we know base type exists in output and we know its type id */
244  NC_CHECK(nc_def_vlen(ogrp, name, obasetype, &vlen_type));
245 
246  return stat;
247 }
248 
249 /*
250  * copy a user-defined opaque type in the group igrp to the group ogrp
251  */
252 static int
253 copy_opaque_type(int igrp, nc_type itype, int ogrp)
254 {
255  int stat = NC_NOERR;
256  nc_type otype;
257  char name[NC_MAX_NAME];
258  size_t size;
259 
260  NC_CHECK(nc_inq_opaque(igrp, itype, name, &size));
261  NC_CHECK(nc_def_opaque(ogrp, size, name, &otype));
262 
263  return stat;
264 }
265 
266 /*
267  * copy a user-defined enum type in the group igrp to the group ogrp
268  */
269 static int
270 copy_enum_type(int igrp, nc_type itype, int ogrp)
271 {
272  int stat = NC_NOERR;
273  nc_type otype;
274  nc_type basetype;
275  size_t basesize;
276  size_t nmembers;
277  char name[NC_MAX_NAME];
278  int i;
279 
280  NC_CHECK(nc_inq_enum(igrp, itype, name, &basetype, &basesize, &nmembers));
281  NC_CHECK(nc_def_enum(ogrp, basetype, name, &otype));
282  for(i = 0; i < nmembers; i++) { /* insert enum members */
283  char ename[NC_MAX_NAME];
284  long long val; /* large enough to hold any integer type */
285  NC_CHECK(nc_inq_enum_member(igrp, itype, i, ename, &val));
286  NC_CHECK(nc_insert_enum(ogrp, otype, ename, &val));
287  }
288  return stat;
289 }
290 
291 /*
292  * copy a user-defined compound type in the group igrp to the group ogrp
293  */
294 static int
295 copy_compound_type(int igrp, nc_type itype, int ogrp)
296 {
297  int stat = NC_NOERR;
298  char name[NC_MAX_NAME];
299  size_t size;
300  size_t nfields;
301  nc_type otype;
302  int fid;
303 
304  NC_CHECK(nc_inq_compound(igrp, itype, name, &size, &nfields));
305  NC_CHECK(nc_def_compound(ogrp, size, name, &otype));
306 
307  for (fid = 0; fid < nfields; fid++) {
308  char fname[NC_MAX_NAME];
309  char ftypename[NC_MAX_NAME];
310  size_t foff;
311  nc_type iftype, oftype;
312  int fndims;
313 
314  NC_CHECK(nc_inq_compound_field(igrp, itype, fid, fname, &foff, &iftype, &fndims, NULL));
315  /* type ids in source don't necessarily correspond to same
316  * typeids in destination, so look up destination typeid by using
317  * field type name */
318  NC_CHECK(nc_inq_type(igrp, iftype, ftypename, NULL));
319  NC_CHECK(nc_inq_typeid(ogrp, ftypename, &oftype));
320  if(fndims == 0) {
321  NC_CHECK(nc_insert_compound(ogrp, otype, fname, foff, oftype));
322  } else { /* field is array type */
323  int *fdimsizes;
324  fdimsizes = (int *) emalloc((fndims + 1) * sizeof(int));
325  stat = nc_inq_compound_field(igrp, itype, fid, NULL, NULL, NULL,
326  NULL, fdimsizes);
327  NC_CHECK(nc_insert_array_compound(ogrp, otype, fname, foff, oftype, fndims, fdimsizes));
328  free(fdimsizes);
329  }
330  }
331  return stat;
332 }
333 
334 
335 /*
336  * copy a user-defined type in the group igrp to the group ogrp
337  */
338 static int
339 copy_type(int igrp, nc_type typeid, int ogrp)
340 {
341  int stat = NC_NOERR;
342  nc_type type_class;
343 
344  NC_CHECK(nc_inq_user_type(igrp, typeid, NULL, NULL, NULL, NULL, &type_class));
345 
346  switch(type_class) {
347  case NC_VLEN:
348  NC_CHECK(copy_vlen_type(igrp, typeid, ogrp));
349  break;
350  case NC_OPAQUE:
351  NC_CHECK(copy_opaque_type(igrp, typeid, ogrp));
352  break;
353  case NC_ENUM:
354  NC_CHECK(copy_enum_type(igrp, typeid, ogrp));
355  break;
356  case NC_COMPOUND:
357  NC_CHECK(copy_compound_type(igrp, typeid, ogrp));
358  break;
359  default:
360  NC_CHECK(NC_EBADTYPE);
361  }
362  return stat;
363 }
364 
365 /* Copy a group and all its subgroups, recursively, from iroot to
366  * oroot, the ncids of input file and output file. This just creates
367  * all the groups in the destination, but doesn't copy anything that's
368  * in the groups yet. */
369 static int
370 copy_groups(int iroot, int oroot)
371 {
372  int stat = NC_NOERR;
373  int numgrps;
374  int *grpids;
375  int i;
376 
377  /* get total number of groups and their ids, including all descendants */
378  NC_CHECK(nc_inq_grps_full(iroot, &numgrps, NULL));
379  grpids = emalloc(numgrps * sizeof(int));
380  NC_CHECK(nc_inq_grps_full(iroot, NULL, grpids));
381  /* create corresponding new groups in ogrp, except for root group */
382  for(i = 1; i < numgrps; i++) {
383  char *grpname_full;
384  char grpname[NC_MAX_NAME];
385  size_t len_name;
386  int ogid, oparid;
387  /* get full group name of input group */
388  NC_CHECK(nc_inq_grpname_full(grpids[i], &len_name, NULL));
389  grpname_full = emalloc(len_name + 1);
390  NC_CHECK(nc_inq_grpname_full(grpids[i], &len_name, grpname_full));
391  /* get id of parent group of corresponding group in output.
392  * Note that this exists, because nc_inq_groups returned
393  * grpids in preorder, so parents are always copied before
394  * their subgroups */
395  NC_CHECK(nc_inq_parid(oroot, grpname_full, &oparid));
396  NC_CHECK(nc_inq_grpname(grpids[i], grpname));
397  /* define corresponding group in output */
398  NC_CHECK(nc_def_grp(oparid, grpname, &ogid));
399  free(grpname_full);
400  }
401  free(grpids);
402  return stat;
403 }
404 
405 /*
406  * Copy the user-defined types in this group (igrp) and all its
407  * subgroups, recursively, to corresponding group in output (ogrp)
408  */
409 static int
410 copy_types(int igrp, int ogrp)
411 {
412  int stat = NC_NOERR;
413  int ntypes;
414  nc_type *types = NULL;
415  int numgrps;
416  int *grpids = NULL;
417  int i;
418 
419  NC_CHECK(nc_inq_typeids(igrp, &ntypes, NULL));
420 
421  if(ntypes > 0) {
422  types = (nc_type *) emalloc(ntypes * sizeof(nc_type));
423  NC_CHECK(nc_inq_typeids(igrp, &ntypes, types));
424  for (i = 0; i < ntypes; i++) {
425  NC_CHECK(copy_type(igrp, types[i], ogrp));
426  }
427  free(types);
428  }
429 
430  /* Copy types from subgroups */
431  NC_CHECK(nc_inq_grps(igrp, &numgrps, NULL));
432  if(numgrps > 0) {
433  grpids = (int *)emalloc(sizeof(int) * numgrps);
434  NC_CHECK(nc_inq_grps(igrp, &numgrps, grpids));
435  for(i = 0; i < numgrps; i++) {
436  int ogid;
437  /* get groupid in output corresponding to grpids[i] in
438  * input, given parent group (or root group) ogrp in
439  * output */
440  NC_CHECK(get_grpid(grpids[i], ogrp, &ogid));
441  NC_CHECK(copy_types(grpids[i], ogid));
442  }
443  free(grpids);
444  }
445  return stat;
446 }
447 
448 /* Copy all netCDF-4 specific variable properties such as chunking,
449  * endianness, deflation, checksumming, fill, etc. */
450 static int
451 copy_var_specials(int igrp, int varid, int ogrp, int o_varid)
452 {
453  int stat = NC_NOERR;
454  { /* handle chunking parameters */
455  int ndims;
456  NC_CHECK(nc_inq_varndims(igrp, varid, &ndims));
457  if (ndims > 0) { /* no chunking for scalar variables */
458  int contig = 0;
459  NC_CHECK(nc_inq_var_chunking(igrp, varid, &contig, NULL));
460  if(contig == 1) {
461  NC_CHECK(nc_def_var_chunking(ogrp, o_varid, NC_CONTIGUOUS, NULL));
462  } else {
463  size_t *chunkp = (size_t *) emalloc(ndims * sizeof(size_t));
464  int *dimids = (int *) emalloc(ndims * sizeof(int));
465  int idim;
466  NC_CHECK(nc_inq_var_chunking(igrp, varid, NULL, chunkp));
467  NC_CHECK(nc_inq_vardimid(igrp, varid, dimids));
468  for(idim = 0; idim < ndims; idim++) {
469  int dimid = dimids[idim];
470  size_t chunksize = chunkspec_size(dimid);
471  if(chunkspec_size(dimid) > 0) { /* found in chunkspec */
472  chunkp[idim] = chunksize;
473  }
474  }
475  /* explicitly set chunking, even if default */
476  NC_CHECK(nc_def_var_chunking(ogrp, o_varid, NC_CHUNKED, chunkp));
477  free(dimids);
478  free(chunkp);
479  }
480  }
481  }
482  { /* handle compression parameters, copying from input, overriding
483  * with command-line options */
484  int shuffle, deflate, deflate_level;
485  NC_CHECK(nc_inq_var_deflate(igrp, varid, &shuffle, &deflate, &deflate_level));
486  if(option_deflate_level >= 0) { /* change output compression, if requested */
487  deflate_level = option_deflate_level;
488  deflate=1;
489  }
490  if(shuffle==0 && option_shuffle_vars != 0) {
491  shuffle = option_shuffle_vars;
492  }
493  if(deflate != 0 || shuffle != 0) {
494  NC_CHECK(nc_def_var_deflate(ogrp, o_varid, shuffle, deflate_level > 0, deflate_level));
495  }
496  }
497  { /* handle checksum parameters */
498  int fletcher32 = 0;
499  NC_CHECK(nc_inq_var_fletcher32(igrp, varid, &fletcher32));
500  if(fletcher32 != 0) {
501  NC_CHECK(nc_def_var_fletcher32(ogrp, o_varid, fletcher32));
502  }
503  }
504  { /* handle endianness */
505  int endianness = 0;
506  NC_CHECK(nc_inq_var_endian(igrp, varid, &endianness));
507  if(endianness != NC_ENDIAN_NATIVE) { /* native is the default */
508  NC_CHECK(nc_def_var_endian(ogrp, o_varid, endianness));
509  }
510  }
511  return stat;
512 }
513 
514 /* Set output variable o_varid (in group ogrp) to use chunking
515  * specified on command line, only called for classic format input and
516  * netCDF-4 format output, so no existing chunk lengths to override. */
517 static int
518 set_var_chunked(int ogrp, int o_varid)
519 {
520  int stat = NC_NOERR;
521  int ndims;
522  int odim;
523  size_t chunk_threshold = CHUNK_THRESHOLD;
524 
525  if(chunkspec_ndims() == 0) /* no chunking specified on command line */
526  return stat;
527  NC_CHECK(nc_inq_varndims(ogrp, o_varid, &ndims));
528 
529  if (ndims > 0) { /* no chunking for scalar variables */
530  int chunked = 0;
531  int *dimids = (int *) emalloc(ndims * sizeof(int));
532  size_t varsize;
533  nc_type vartype;
534  size_t value_size;
535  int is_unlimited = 0;
536 
537  NC_CHECK(nc_inq_vardimid (ogrp, o_varid, dimids));
538  NC_CHECK(nc_inq_vartype(ogrp, o_varid, &vartype));
539  /* from type, get size in memory needed for each value */
540  NC_CHECK(nc_inq_type(ogrp, vartype, NULL, &value_size));
541  varsize = value_size;
542 
543  /* Determine if this variable should be chunked. A variable
544  * should be chunked if any of its dims are in command-line
545  * chunk spec. It will also be chunked if any of its
546  * dims are unlimited. */
547  for(odim = 0; odim < ndims; odim++) {
548  int odimid = dimids[odim];
549  int idimid = dimmap_idimid(odimid); /* corresponding dimid in input file */
550  if(dimmap_ounlim(odimid))
551  is_unlimited = 1;
552  if(idimid != -1) {
553  size_t chunksize = chunkspec_size(idimid); /* from chunkspec */
554  size_t dimlen;
555  NC_CHECK(nc_inq_dimlen(ogrp, odimid, &dimlen));
556  if( (chunksize > 0) || dimmap_ounlim(odimid)) {
557  chunked = 1;
558  }
559  varsize *= dimlen;
560  }
561  }
562  /* Don't chunk small variables that don't use an unlimited
563  * dimension. */
564  if(varsize < chunk_threshold && !is_unlimited)
565  chunked = 0;
566 
567  if(chunked) {
568  /* Allocate chunksizes and set defaults to dimsize for any
569  * dimensions not mentioned in chunkspec. */
570  size_t *chunkp = (size_t *) emalloc(ndims * sizeof(size_t));
571  for(odim = 0; odim < ndims; odim++) {
572  int odimid = dimids[odim];
573  int idimid = dimmap_idimid(odimid);
574  size_t chunksize = chunkspec_size(idimid);
575  if(chunksize > 0) {
576  chunkp[odim] = chunksize;
577  } else {
578  NC_CHECK(nc_inq_dimlen(ogrp, odimid, &chunkp[odim]));
579  }
580  }
581  NC_CHECK(nc_def_var_chunking(ogrp, o_varid, NC_CHUNKED, chunkp));
582  free(chunkp);
583  }
584  free(dimids);
585  }
586  return stat;
587 }
588 
589 /* Set variable to compression specified on command line */
590 static int
591 set_var_compressed(int ogrp, int o_varid)
592 {
593  int stat = NC_NOERR;
594  if (option_deflate_level >= 0) {
595  int deflate = 1;
596  NC_CHECK(nc_def_var_deflate(ogrp, o_varid, option_shuffle_vars, deflate, option_deflate_level));
597  }
598  return stat;
599 }
600 
601 /* Release the variable chunk cache allocated for variable varid in
602  * group grp. This is not necessary, but will save some memory when
603  * processing one variable at a time. */
604 #ifdef UNUSED
605 static int
606 free_var_chunk_cache(int grp, int varid)
607 {
608  int stat = NC_NOERR;
609  size_t chunk_cache_size = 1;
610  size_t cache_nelems = 1;
611  float cache_preemp = 0;
612  int kind;
613  NC_CHECK(nc_inq_format(grp, &kind));
614  if(kind == NC_FORMAT_NETCDF4 || kind == NC_FORMAT_NETCDF4_CLASSIC) {
615  int contig = 1;
616  NC_CHECK(nc_inq_var_chunking(grp, varid, &contig, NULL));
617  if(contig == 0) { /* chunked */
618  NC_CHECK(nc_set_var_chunk_cache(grp, varid, chunk_cache_size, cache_nelems, cache_preemp));
619  }
620  }
621  return stat;
622 }
623 #endif
624 
625 #endif /* USE_NETCDF4 */
626 
627 /* Copy dimensions from group igrp to group ogrp, also associate input
628  * dimids with output dimids (they need not match, because the input
629  * dimensions may have been defined in a different order than we define
630  * the output dimensions here. */
631 static int
632 copy_dims(int igrp, int ogrp)
633 {
634  int stat = NC_NOERR;
635  int ndims;
636  int dgrp;
637 #ifdef USE_NETCDF4
638  int nunlims;
639  int *dimids;
640  int *unlimids;
641 #else
642  int unlimid;
643 #endif /* USE_NETCDF4 */
644 
645  NC_CHECK(nc_inq_ndims(igrp, &ndims));
646 
647 #ifdef USE_NETCDF4
648  /* In netCDF-4 files, dimids may not be sequential because they
649  * may be defined in various groups, and we are only looking at one
650  * group at a time. */
651  /* Find the dimension ids in this group, don't include parents. */
652  dimids = (int *) emalloc((ndims + 1) * sizeof(int));
653  NC_CHECK(nc_inq_dimids(igrp, NULL, dimids, 0));
654  /* Find the number of unlimited dimensions and get their IDs */
655  NC_CHECK(nc_inq_unlimdims(igrp, &nunlims, NULL));
656  unlimids = (int *) emalloc((nunlims + 1) * sizeof(int));
657  NC_CHECK(nc_inq_unlimdims(igrp, NULL, unlimids));
658 #else
659  NC_CHECK(nc_inq_unlimdim(igrp, &unlimid));
660 #endif /* USE_NETCDF4 */
661 
662  /* Copy each dimension to output, including unlimited dimension(s) */
663  for (dgrp = 0; dgrp < ndims; dgrp++) {
664  char name[NC_MAX_NAME];
665  size_t length;
666  int i_is_unlim;
667  int o_is_unlim;
668  int idimid, odimid;
669 #ifdef USE_NETCDF4
670  int uld;
671 #endif
672 
673  i_is_unlim = 0;
674 #ifdef USE_NETCDF4
675  idimid = dimids[dgrp];
676  for (uld = 0; uld < nunlims; uld++) {
677  if(idimid == unlimids[uld]) {
678  i_is_unlim = 1;
679  break;
680  }
681  }
682 #else
683  idimid = dgrp;
684  if(unlimid != -1 && (idimid == unlimid)) {
685  i_is_unlim = 1;
686  }
687 #endif /* USE_NETCDF4 */
688 
689  stat = nc_inq_dim(igrp, idimid, name, &length);
690  if (stat == NC_EDIMSIZE && sizeof(size_t) < 8) {
691  error("dimension \"%s\" requires 64-bit platform", name);
692  }
693  NC_CHECK(stat);
694  o_is_unlim = i_is_unlim;
695  if(i_is_unlim && !option_fix_unlimdims) {
696  NC_CHECK(nc_def_dim(ogrp, name, NC_UNLIMITED, &odimid));
697  } else {
698  NC_CHECK(nc_def_dim(ogrp, name, length, &odimid));
699  o_is_unlim = 0;
700  }
701  /* Store (idimid, odimid) mapping for later use, also whether unlimited */
702  dimmap_store(idimid, odimid, i_is_unlim, o_is_unlim);
703  }
704 #ifdef USE_NETCDF4
705  free(dimids);
706  free(unlimids);
707 #endif /* USE_NETCDF4 */
708  return stat;
709 }
710 
711 /* Copy the attributes for variable ivar in group igrp to variable
712  * ovar in group ogrp. Global (group) attributes are specified by
713  * using the varid NC_GLOBAL */
714 static int
715 copy_atts(int igrp, int ivar, int ogrp, int ovar)
716 {
717  int natts;
718  int iatt;
719  int stat = NC_NOERR;
720 
721  NC_CHECK(nc_inq_varnatts(igrp, ivar, &natts));
722 
723  for(iatt = 0; iatt < natts; iatt++) {
724  char name[NC_MAX_NAME];
725  NC_CHECK(nc_inq_attname(igrp, ivar, iatt, name));
726  NC_CHECK(nc_copy_att(igrp, ivar, name, ogrp, ovar));
727  }
728  return stat;
729 }
730 
731 /* copy the schema for a single variable in group igrp to group ogrp */
732 static int
733 copy_var(int igrp, int varid, int ogrp)
734 {
735  int stat = NC_NOERR;
736  int ndims;
737  int *idimids; /* ids of dims for input variable */
738  int *odimids; /* ids of dims for output variable */
739  char name[NC_MAX_NAME];
740  nc_type typeid, o_typeid;
741  int natts;
742  int i;
743  int o_varid;
744 
745  NC_CHECK(nc_inq_varndims(igrp, varid, &ndims));
746  idimids = (int *) emalloc((ndims + 1) * sizeof(int));
747  NC_CHECK(nc_inq_var(igrp, varid, name, &typeid, NULL, idimids, &natts));
748  o_typeid = typeid;
749 #ifdef USE_NETCDF4
750  if (typeid > NC_STRING) { /* user-defined type */
751  /* type ids in source don't necessarily correspond to same
752  * typeids in destination, so look up destination typeid by
753  * using type name */
754  char type_name[NC_MAX_NAME];
755  NC_CHECK(nc_inq_type(igrp, typeid, type_name, NULL));
756  NC_CHECK(nc_inq_typeid(ogrp, type_name, &o_typeid));
757  }
758 #endif /* USE_NETCDF4 */
759 
760  /* get the corresponding dimids in the output file */
761  odimids = (int *) emalloc((ndims + 1) * sizeof(int));
762  for(i = 0; i < ndims; i++) {
763  odimids[i] = dimmap_odimid(idimids[i]);
764  if(odimids[i] == -1) {
765  error("Oops, no dimension in output associated with input dimid %d", idimids[i]);
766  }
767  }
768 
769  /* define the output variable */
770  NC_CHECK(nc_def_var(ogrp, name, o_typeid, ndims, odimids, &o_varid));
771 
772  /* attach the variable attributes to the output variable */
773  NC_CHECK(copy_atts(igrp, varid, ogrp, o_varid));
774 #ifdef USE_NETCDF4
775  {
776  int inkind;
777  int outkind;
778  NC_CHECK(nc_inq_format(igrp, &inkind));
779  NC_CHECK(nc_inq_format(ogrp, &outkind));
780  if(outkind == NC_FORMAT_NETCDF4 || outkind == NC_FORMAT_NETCDF4_CLASSIC) {
781  if((inkind == NC_FORMAT_NETCDF4 || inkind == NC_FORMAT_NETCDF4_CLASSIC)) {
782  /* Copy all netCDF-4 specific variable properties such as
783  * chunking, endianness, deflation, checksumming, fill, etc. */
784  NC_CHECK(copy_var_specials(igrp, varid, ogrp, o_varid));
785  } else {
786  /* Set chunking if specified in command line option */
787  NC_CHECK(set_var_chunked(ogrp, o_varid));
788  }
789  /* Set compression if specified in command line option */
790  NC_CHECK(set_var_compressed(ogrp, o_varid));
791  }
792  }
793 #endif /* USE_NETCDF4 */
794  free(idimids);
795  free(odimids);
796  return stat;
797 }
798 
799 /* copy the schema for all the variables in group igrp to group ogrp */
800 static int
801 copy_vars(int igrp, int ogrp)
802 {
803  int stat = NC_NOERR;
804  int nvars;
805  int varid;
806 
807  NC_CHECK(nc_inq_nvars(igrp, &nvars));
808  for (varid = 0; varid < nvars; varid++) {
809  NC_CHECK(copy_var(igrp, varid, ogrp));
810  }
811  return stat;
812 }
813 
814 /* Copy the schema in a group and all its subgroups, recursively, from
815  * group igrp in input to parent group ogrp in destination. Use
816  * dimmap array to map input dimids to output dimids. */
817 static int
818 copy_schema(int igrp, int ogrp)
819 {
820  int stat = NC_NOERR;
821  int ogid; /* like igrp but in output file */
822 
823  /* get groupid in output corresponding to group igrp in input,
824  * given parent group (or root group) ogrp in output */
825  NC_CHECK(get_grpid(igrp, ogrp, &ogid));
826 
827  NC_CHECK(copy_dims(igrp, ogid));
828  NC_CHECK(copy_atts(igrp, NC_GLOBAL, ogid, NC_GLOBAL));
829  NC_CHECK(copy_vars(igrp, ogid));
830 #ifdef USE_NETCDF4
831  {
832  int numgrps;
833  int *grpids;
834  int i;
835  /* Copy schema from subgroups */
836  stat = nc_inq_grps(igrp, &numgrps, NULL);
837  grpids = (int *)emalloc((numgrps + 1) * sizeof(int));
838  NC_CHECK(nc_inq_grps(igrp, &numgrps, grpids));
839 
840  for(i = 0; i < numgrps; i++) {
841  NC_CHECK(copy_schema(grpids[i], ogid));
842  }
843  free(grpids);
844  }
845 #endif /* USE_NETCDF4 */
846  return stat;
847 }
848 
849 /* Return number of values for a variable varid in a group igrp */
850 static int
851 inq_nvals(int igrp, int varid, long long *nvalsp) {
852  int stat = NC_NOERR;
853  int ndims;
854  int *dimids;
855  int dim;
856  long long nvals = 1;
857 
858  NC_CHECK(nc_inq_varndims(igrp, varid, &ndims));
859  dimids = (int *) emalloc((ndims + 1) * sizeof(int));
860  NC_CHECK(nc_inq_vardimid (igrp, varid, dimids));
861  for(dim = 0; dim < ndims; dim++) {
862  size_t len;
863  NC_CHECK(nc_inq_dimlen(igrp, dimids[dim], &len));
864  nvals *= len;
865  }
866  if(nvalsp)
867  *nvalsp = nvals;
868  free(dimids);
869  return stat;
870 }
871 
872 /* Copy data from variable varid in group igrp to corresponding group
873  * ogrp. */
874 static int
875 copy_var_data(int igrp, int varid, int ogrp) {
876  int stat = NC_NOERR;
877  nc_type vartype;
878  long long nvalues; /* number of values for this variable */
879  size_t ntoget; /* number of values to access this iteration */
880  size_t value_size; /* size of a single value of this variable */
881  static void *buf = 0; /* buffer for the variable values */
882  char varname[NC_MAX_NAME];
883  int ovarid;
884  size_t *start;
885  size_t *count;
886  nciter_t *iterp; /* opaque structure for iteration status */
887  int do_realloc = 0;
888 #ifdef USE_NETCDF4
889  int okind;
890  size_t chunksize;
891 #endif
892 
893  NC_CHECK(inq_nvals(igrp, varid, &nvalues));
894  if(nvalues == 0)
895  return stat;
896  /* get corresponding output variable */
897  NC_CHECK(nc_inq_varname(igrp, varid, varname));
898  NC_CHECK(nc_inq_varid(ogrp, varname, &ovarid));
899  NC_CHECK(nc_inq_vartype(igrp, varid, &vartype));
900  /* from type, get size in memory needed for each value */
901  NC_CHECK(nc_inq_type(igrp, vartype, NULL, &value_size));
902  if(value_size > option_copy_buffer_size) {
903  option_copy_buffer_size = value_size;
904  do_realloc = 1;
905  }
906 #ifdef USE_NETCDF4
907  NC_CHECK(nc_inq_format(ogrp, &okind));
908  if(okind == NC_FORMAT_NETCDF4 || okind == NC_FORMAT_NETCDF4_CLASSIC) {
909  /* if this variable chunked, set variable chunk cache size */
910  int contig = 1;
911  NC_CHECK(nc_inq_var_chunking(ogrp, ovarid, &contig, NULL));
912  if(contig == 0) { /* chunked */
913  if(option_compute_chunkcaches) {
914  /* Try to estimate variable-specific chunk cache,
915  * depending on specific size and shape of this
916  * variable's chunks. This doesn't work yet. */
917  size_t chunkcache_size, chunkcache_nelems;
918  float chunkcache_preemption;
919  NC_CHECK(inq_var_chunking_params(igrp, varid, ogrp, ovarid,
920  &chunkcache_size,
921  &chunkcache_nelems,
922  &chunkcache_preemption));
923  NC_CHECK(nc_set_var_chunk_cache(ogrp, ovarid,
924  chunkcache_size,
925  chunkcache_nelems,
926  chunkcache_preemption));
927  } else {
928  /* by default, use same chunk cache for all chunked variables */
929  NC_CHECK(nc_set_var_chunk_cache(ogrp, ovarid,
930  option_chunk_cache_size,
931  option_chunk_cache_nelems,
932  COPY_CHUNKCACHE_PREEMPTION));
933  }
934  }
935  }
936  /* For chunked variables, option_copy_buffer_size must also be at least as large as
937  * size of a chunk in input, otherwise resize it. */
938  {
939  NC_CHECK(inq_var_chunksize(igrp, varid, &chunksize));
940  if(chunksize > option_copy_buffer_size) {
941  option_copy_buffer_size = chunksize;
942  do_realloc = 1;
943  }
944  }
945 #endif /* USE_NETCDF4 */
946  if(buf && do_realloc) {
947  free(buf);
948  buf = 0;
949  }
950  if(buf == 0) { /* first time or needs to grow */
951  buf = emalloc(option_copy_buffer_size);
952  memset((void*)buf,0,option_copy_buffer_size);
953  }
954 
955  /* initialize variable iteration */
956  NC_CHECK(nc_get_iter(igrp, varid, option_copy_buffer_size, &iterp));
957 
958  start = (size_t *) emalloc((iterp->rank + 1) * sizeof(size_t));
959  count = (size_t *) emalloc((iterp->rank + 1) * sizeof(size_t));
960  /* nc_next_iter() initializes start and count on first call,
961  * changes start and count to iterate through whole variable on
962  * subsequent calls. */
963  while((ntoget = nc_next_iter(iterp, start, count)) > 0) {
964  NC_CHECK(nc_get_vara(igrp, varid, start, count, buf));
965  NC_CHECK(nc_put_vara(ogrp, ovarid, start, count, buf));
966 #ifdef USE_NETCDF4
967  /* we have to explicitly free values for strings and vlens */
968  if(vartype == NC_STRING) {
969  NC_CHECK(nc_free_string(ntoget, (char **)buf));
970  } else if(vartype > NC_STRING) { /* user-defined type */
971  nc_type vclass;
972  NC_CHECK(nc_inq_user_type(igrp, vartype, NULL, NULL, NULL, NULL, &vclass));
973  if(vclass == NC_VLEN) {
974  NC_CHECK(nc_free_vlens(ntoget, (nc_vlen_t *)buf));
975  }
976  }
977 #endif /* USE_NETCDF4 */
978  } /* end main iteration loop */
979 #ifdef USE_NETCDF4
980  /* We're all done with this input and output variable, so if
981  * either variable is chunked, free up its variable chunk cache */
982  /* NC_CHECK(free_var_chunk_cache(igrp, varid)); */
983  /* NC_CHECK(free_var_chunk_cache(ogrp, ovarid)); */
984 #endif /* USE_NETCDF4 */
985  free(start);
986  free(count);
987  NC_CHECK(nc_free_iter(iterp));
988  return stat;
989 }
990 
991 /* Copy data from variables in group igrp to variables in
992  * corresponding group with parent ogrp, and all subgroups
993  * recursively */
994 static int
995 copy_data(int igrp, int ogrp)
996 {
997  int stat = NC_NOERR;
998  int ogid;
999  int nvars;
1000  int varid;
1001 #ifdef USE_NETCDF4
1002  int numgrps;
1003  int *grpids;
1004  int i;
1005 #endif
1006 
1007  /* get groupid in output corresponding to group igrp in input,
1008  * given parent group (or root group) ogrp in output */
1009  NC_CHECK(get_grpid(igrp, ogrp, &ogid));
1010 
1011  /* Copy data from this group */
1012  NC_CHECK(nc_inq_nvars(igrp, &nvars));
1013 
1014  for (varid = 0; varid < nvars; varid++) {
1015  NC_CHECK(copy_var_data(igrp, varid, ogid));
1016  }
1017 #ifdef USE_NETCDF4
1018  /* Copy data from subgroups */
1019  stat = nc_inq_grps(igrp, &numgrps, NULL);
1020  grpids = (int *)emalloc((numgrps + 1) * sizeof(int));
1021  NC_CHECK(nc_inq_grps(igrp, &numgrps, grpids));
1022 
1023  for(i = 0; i < numgrps; i++) {
1024  NC_CHECK(copy_data(grpids[i], ogid));
1025  }
1026  free(grpids);
1027 #endif /* USE_NETCDF4 */
1028  return stat;
1029 }
1030 
1031 /* Count total number of dimensions in ncid and all its subgroups */
1032 int
1033 count_dims(ncid) {
1034  int numgrps;
1035  int *grpids;
1036  int igrp;
1037  int ndims=0;
1038  /* get total number of groups and their ids, including all descendants */
1039  NC_CHECK(nc_inq_grps_full(ncid, &numgrps, NULL));
1040  grpids = emalloc(numgrps * sizeof(int));
1041  NC_CHECK(nc_inq_grps_full(ncid, NULL, grpids));
1042  for(igrp = 0; igrp < numgrps; igrp++) {
1043  int ndims_local;
1044  nc_inq_ndims(grpids[igrp], &ndims_local);
1045  ndims += ndims_local;
1046  }
1047  free(grpids);
1048  return ndims;
1049 }
1050 
1051 /* Test if special case: netCDF-3 file with more than one record
1052  * variable. Performance can be very slow for this case when the disk
1053  * block size is large, there are many record variables, and a
1054  * record's worth of data for some variables is smaller than the disk
1055  * block size. In this case, copying the record variables a variable
1056  * at a time causes much rereading of record data, so instead we want
1057  * to copy data a record at a time. */
1058 static int
1059 nc3_special_case(int ncid, int kind) {
1060  if (kind == NC_FORMAT_CLASSIC || kind == NC_FORMAT_64BIT) {
1061  int recdimid = 0;
1062  NC_CHECK(nc_inq_unlimdim(ncid, &recdimid));
1063  if (recdimid != -1) { /* we have a record dimension */
1064  int nvars;
1065  int varid;
1066  NC_CHECK(nc_inq_nvars(ncid, &nvars));
1067  for (varid = 0; varid < nvars; varid++) {
1068  int *dimids = 0;
1069  int ndims;
1070  NC_CHECK( nc_inq_varndims(ncid, varid, &ndims) );
1071  if (ndims > 0) {
1072  int dimids0;
1073  dimids = (int *) emalloc((ndims + 1) * sizeof(int));
1074  NC_CHECK( nc_inq_vardimid(ncid, varid, dimids) );
1075  dimids0 = dimids[0];
1076  free(dimids);
1077  if(dimids0 == recdimid) {
1078  return 1; /* found a record variable */
1079  }
1080  }
1081  }
1082  }
1083  }
1084  return 0;
1085 }
1086 
1087 /* Classify variables in ncid as either fixed-size variables (with no
1088  * unlimited dimension) or as record variables (with an unlimited
1089  * dimension) */
1090 static int
1091 classify_vars(
1092  int ncid, /* netCDF ID */
1093  size_t *nf, /* for returning number of fixed-size variables */
1094  int **fvars, /* the array of fixed_size variable IDS, caller should free */
1095  size_t *nr, /* for returning number of record variables */
1096  int **rvars) /* the array of record variable IDs, caller should free */
1097 {
1098  int varid;
1099  int nvars;
1100  NC_CHECK(nc_inq_nvars(ncid, &nvars));
1101  *nf = 0;
1102  *fvars = (int *) emalloc(nvars * sizeof(int));
1103  *nr = 0;
1104  *rvars = (int *) emalloc(nvars * sizeof(int));
1105  for (varid = 0; varid < nvars; varid++) {
1106  if (isrecvar(ncid, varid)) {
1107  (*rvars)[*nr] = varid;
1108  (*nr)++;
1109  } else {
1110  (*fvars)[*nf] = varid;
1111  (*nf)++;
1112  }
1113  }
1114  return NC_NOERR;
1115 }
1116 
1117 /* Only called for classic format or 64-bit offset format files, to speed up special case */
1118 static int
1119 copy_fixed_size_data(int igrp, int ogrp, size_t nfixed_vars, int *fixed_varids) {
1120  size_t ivar;
1121  /* for each fixed-size variable, copy data */
1122  for (ivar = 0; ivar < nfixed_vars; ivar++) {
1123  int varid = fixed_varids[ivar];
1124  NC_CHECK(copy_var_data(igrp, varid, ogrp));
1125  }
1126  if (fixed_varids)
1127  free(fixed_varids);
1128  return NC_NOERR;
1129 }
1130 
1131 /* copy a record's worth of data for a variable from input to output */
1132 static int
1133 copy_rec_var_data(int ncid, /* input */
1134  int ogrp, /* output */
1135  int irec, /* record number */
1136  int varid, /* input variable id */
1137  int ovarid, /* output variable id */
1138  size_t *start, /* start indices for record data */
1139  size_t *count, /* edge lengths for record data */
1140  void *buf /* buffer large enough to hold data */
1141  )
1142 {
1143  NC_CHECK(nc_get_vara(ncid, varid, start, count, buf));
1144  NC_CHECK(nc_put_vara(ogrp, ovarid, start, count, buf));
1145  return NC_NOERR;
1146 }
1147 
1148 /* Only called for classic format or 64-bit offset format files, to speed up special case */
1149 static int
1150 copy_record_data(int ncid, int ogrp, size_t nrec_vars, int *rec_varids) {
1151  int unlimid;
1152  size_t nrecs = 0; /* how many records? */
1153  size_t irec;
1154  size_t ivar;
1155  void **buf; /* space for reading in data for each variable */
1156  int *rec_ovarids; /* corresponding varids in output */
1157  size_t **start;
1158  size_t **count;
1159  NC_CHECK(nc_inq_unlimdim(ncid, &unlimid));
1160  NC_CHECK(nc_inq_dimlen(ncid, unlimid, &nrecs));
1161  buf = (void **) emalloc(nrec_vars * sizeof(void *));
1162  rec_ovarids = (int *) emalloc(nrec_vars * sizeof(int));
1163  start = (size_t **) emalloc(nrec_vars * sizeof(size_t*));
1164  count = (size_t **) emalloc(nrec_vars * sizeof(size_t*));
1165  /* get space to hold one record's worth of data for each record variable */
1166  for (ivar = 0; ivar < nrec_vars; ivar++) {
1167  int varid;
1168  int ndims;
1169  int *dimids;
1170  nc_type vartype;
1171  size_t value_size;
1172  int dimid;
1173  int ii;
1174  size_t nvals;
1175  char varname[NC_MAX_NAME];
1176  varid = rec_varids[ivar];
1177  NC_CHECK(nc_inq_varndims(ncid, varid, &ndims));
1178  dimids = (int *) emalloc((1 + ndims) * sizeof(int));
1179  start[ivar] = (size_t *) emalloc(ndims * sizeof(size_t));
1180  count[ivar] = (size_t *) emalloc(ndims * sizeof(size_t));
1181  NC_CHECK(nc_inq_vardimid (ncid, varid, dimids));
1182  NC_CHECK(nc_inq_vartype(ncid, varid, &vartype));
1183  NC_CHECK(nc_inq_type(ncid, vartype, NULL, &value_size));
1184  nvals = 1;
1185  for(ii = 1; ii < ndims; ii++) { /* for rec size, don't include first record dimension */
1186  size_t dimlen;
1187  dimid = dimids[ii];
1188  NC_CHECK(nc_inq_dimlen(ncid, dimid, &dimlen));
1189  nvals *= dimlen;
1190  start[ivar][ii] = 0;
1191  count[ivar][ii] = dimlen;
1192  }
1193  start[ivar][0] = 0;
1194  count[ivar][0] = 1; /* 1 record */
1195  buf[ivar] = (void *) emalloc(nvals * value_size);
1196  NC_CHECK(nc_inq_varname(ncid, varid, varname));
1197  NC_CHECK(nc_inq_varid(ogrp, varname, &rec_ovarids[ivar]));
1198  if(dimids)
1199  free(dimids);
1200  }
1201 
1202  /* for each record, copy all variable data */
1203  for(irec = 0; irec < nrecs; irec++) {
1204  for (ivar = 0; ivar < nrec_vars; ivar++) {
1205  int varid, ovarid;
1206  varid = rec_varids[ivar];
1207  ovarid = rec_ovarids[ivar];
1208  start[ivar][0] = irec;
1209  NC_CHECK(copy_rec_var_data(ncid, ogrp, irec, varid, ovarid,
1210  start[ivar], count[ivar], buf[ivar]));
1211  }
1212  }
1213  for (ivar = 0; ivar < nrec_vars; ivar++) {
1214  if(start[ivar])
1215  free(start[ivar]);
1216  if(count[ivar])
1217  free(count[ivar]);
1218  }
1219  if(start)
1220  free(start);
1221  if(count)
1222  free(count);
1223  for (ivar = 0; ivar < nrec_vars; ivar++) {
1224  if(buf[ivar]) {
1225  free(buf[ivar]);
1226  }
1227  }
1228  if (rec_varids)
1229  free(rec_varids);
1230  if(buf)
1231  free(buf);
1232  if(rec_ovarids)
1233  free(rec_ovarids);
1234  return NC_NOERR;
1235 }
1236 
1237 /* copy infile to outfile using netCDF API
1238  */
1239 static int
1240 copy(char* infile, char* outfile)
1241 {
1242  int stat = NC_NOERR;
1243  int igrp, ogrp;
1244  int inkind, outkind;
1245  int open_mode = NC_NOWRITE;
1246  int create_mode = NC_CLOBBER;
1247  size_t ndims;
1248 
1249  if(option_read_diskless) {
1250  open_mode |= NC_DISKLESS;
1251  }
1252 
1253  NC_CHECK(nc_open(infile, open_mode, &igrp));
1254 
1255  NC_CHECK(nc_inq_format(igrp, &inkind));
1256 
1257 /* option_kind specifies which netCDF format for output:
1258  * -1 -> same as input,
1259  * 1 -> classic
1260  * 2 -> 64-bit offset
1261  * 3 -> netCDF-4,
1262  * 4 -> netCDF-4 classic model
1263  *
1264  * However, if compression or shuffling was specified and kind was -1,
1265  * kind is changed to format 4 that supports compression for input of
1266  * type 1 or 2.
1267  */
1268  outkind = option_kind;
1269  if (option_kind == SAME_AS_INPUT) { /* default, kind not specified */
1270  outkind = inkind;
1271  /* Deduce output kind if netCDF-4 features requested */
1272  if (inkind == NC_FORMAT_CLASSIC || inkind == NC_FORMAT_64BIT) {
1273  if (option_deflate_level > 0 ||
1274  option_shuffle_vars == NC_SHUFFLE ||
1275  option_chunkspec)
1276  {
1277  outkind = NC_FORMAT_NETCDF4_CLASSIC;
1278  }
1279  }
1280  }
1281 
1282 #ifdef USE_NETCDF4
1283  if(option_chunkspec) {
1284  /* Now that input is open, can parse option_chunkspec into binary
1285  * structure. */
1286  NC_CHECK(chunkspec_parse(igrp, option_chunkspec));
1287  }
1288 #endif /* USE_NETCDF4 */
1289 
1290  if(option_write_diskless)
1291  create_mode |= NC_WRITE | NC_DISKLESS; /* NC_WRITE persists diskless file on close */
1292  switch(outkind) {
1293  case NC_FORMAT_CLASSIC:
1294  /* nothing to do */
1295  break;
1296  case NC_FORMAT_64BIT:
1297  create_mode |= NC_64BIT_OFFSET;
1298  break;
1299 #ifdef USE_NETCDF4
1300  case NC_FORMAT_NETCDF4:
1301  create_mode |= NC_NETCDF4;
1302  break;
1304  create_mode |= NC_NETCDF4 | NC_CLASSIC_MODEL;
1305  break;
1306 #else
1307  case NC_FORMAT_NETCDF4:
1309  error("nccopy built with --disable-netcdf4, can't create netCDF-4 files");
1310  break;
1311 #endif /* USE_NETCDF4 */
1312  default:
1313  error("bad value (%d) for -k option\n", option_kind);
1314  break;
1315  }
1316  NC_CHECK(nc_create(outfile, create_mode, &ogrp));
1317  NC_CHECK(nc_set_fill(ogrp, NC_NOFILL, NULL));
1318 
1319 #ifdef USE_NETCDF4
1320  /* Because types in one group may depend on types in a different
1321  * group, need to create all groups before defining types */
1322  if(inkind == NC_FORMAT_NETCDF4) {
1323  NC_CHECK(copy_groups(igrp, ogrp));
1324  NC_CHECK(copy_types(igrp, ogrp));
1325  }
1326 #endif /* USE_NETCDF4 */
1327 
1328  ndims = count_dims(igrp);
1329  NC_CHECK(dimmap_init(ndims));
1330  NC_CHECK(copy_schema(igrp, ogrp));
1331  NC_CHECK(nc_enddef(ogrp));
1332 
1333  /* For performance, special case netCDF-3 input or output file with record
1334  * variables, to copy a record-at-a-time instead of a
1335  * variable-at-a-time. */
1336  if(nc3_special_case(igrp, inkind)) {
1337  size_t nfixed_vars, nrec_vars;
1338  int *fixed_varids;
1339  int *rec_varids;
1340  NC_CHECK(classify_vars(igrp, &nfixed_vars, &fixed_varids, &nrec_vars, &rec_varids));
1341  NC_CHECK(copy_fixed_size_data(igrp, ogrp, nfixed_vars, fixed_varids));
1342  NC_CHECK(copy_record_data(igrp, ogrp, nrec_vars, rec_varids));
1343  } else if (nc3_special_case(ogrp, outkind)) {
1344  size_t nfixed_vars, nrec_vars;
1345  int *fixed_varids;
1346  int *rec_varids;
1347  /* classifies output vars, but returns input varids */
1348  NC_CHECK(classify_vars(ogrp, &nfixed_vars, &fixed_varids, &nrec_vars, &rec_varids));
1349  NC_CHECK(copy_fixed_size_data(igrp, ogrp, nfixed_vars, fixed_varids));
1350  NC_CHECK(copy_record_data(igrp, ogrp, nrec_vars, rec_varids));
1351  } else {
1352  NC_CHECK(copy_data(igrp, ogrp)); /* recursive, to handle nested groups */
1353  }
1354 
1355  NC_CHECK(nc_close(igrp));
1356  NC_CHECK(nc_close(ogrp));
1357  return stat;
1358 }
1359 
1360 static void
1361 usage(void)
1362 {
1363 #define USAGE "\
1364  [-k n] specify kind of netCDF format for output file, default same as input\n\
1365  1 classic, 2 64-bit offset, 3 netCDF-4, 4 netCDF-4 classic model\n\
1366  [-d n] set deflation compression level, default same as input (0=none 9=max)\n\
1367  [-s] add shuffle option to deflation compression\n\
1368  [-c chunkspec] specify chunking for dimensions, e.g. \"dim1/N1,dim2/N2,...\"\n\
1369  [-u] convert unlimited dimensions to fixed-size dimensions in output copy\n\
1370  [-w] write whole output file from diskless netCDF on close\n\
1371  [-m n] set size in bytes of copy buffer, default is 5000000 bytes\n\
1372  [-h n] set size in bytes of chunk_cache for chunked variables\n\
1373  [-e n] set number of elements that chunk_cache can hold\n\
1374  [-r] read whole input file into diskless file on open (classic or 64-bit offset format only)\n\
1375  infile name of netCDF input file\n\
1376  outfile name for netCDF output file\n"
1377 
1378  /* Don't document this flaky option until it works better */
1379  /* [-x] use experimental computed estimates for variable-specific chunk caches\n\ */
1380 
1381  error("%s [-k n] [-d n] [-s] [-c chunkspec] [-u] [-w] [-m n] [-h n] [-e n] [-r] infile outfile\n%s",
1382  progname, USAGE);
1383 }
1384 
1634 int
1635 main(int argc, char**argv)
1636 {
1637  char* inputfile = NULL;
1638  char* outputfile = NULL;
1639  int c;
1640 
1641 /* table of formats for legal -k values */
1642  struct Kvalues {
1643  char* name;
1644  int kind;
1645  } legalkinds[] = {
1646  {"1", NC_FORMAT_CLASSIC},
1647  {"classic", NC_FORMAT_CLASSIC},
1648 
1649  /* The 64-bit offset kind (2) */
1650  {"2", NC_FORMAT_64BIT},
1651  {"64-bit-offset", NC_FORMAT_64BIT},
1652  {"64-bit offset", NC_FORMAT_64BIT},
1653 
1654  /* NetCDF-4 HDF5 format */
1655  {"3", NC_FORMAT_NETCDF4},
1656  {"hdf5", NC_FORMAT_NETCDF4},
1657  {"netCDF-4", NC_FORMAT_NETCDF4},
1658  {"netCDF4", NC_FORMAT_NETCDF4},
1659  {"enhanced", NC_FORMAT_NETCDF4},
1660 
1661  /* NetCDF-4 HDF5 format, but using only nc3 data model */
1663  {"hdf5-nc3", NC_FORMAT_NETCDF4_CLASSIC},
1664  {"netCDF-4 classic model", NC_FORMAT_NETCDF4_CLASSIC},
1665  {"netCDF4_classic", NC_FORMAT_NETCDF4_CLASSIC},
1666  {"enhanced-nc3", NC_FORMAT_NETCDF4_CLASSIC},
1667 
1668  /* null terminate*/
1669  {NULL,0}
1670  };
1671 
1672  opterr = 1;
1673  progname = argv[0];
1674 
1675  if (argc <= 1)
1676  {
1677  usage();
1678  }
1679 
1680  while ((c = getopt(argc, argv, "k:d:sum:c:h:e:rwx")) != -1) {
1681  switch(c) {
1682  case 'k': /* for specifying variant of netCDF format to be generated
1683  Possible values are:
1684  1 (=> classic 32 bit)
1685  2 (=> classic 64 bit offsets)
1686  3 (=> netCDF-4/HDF5)
1687  4 (=> classic, but stored in netCDF-4/HDF5 format)
1688  Also allow string versions of above
1689  "classic"
1690  "64-bit-offset"
1691  "64-bit offset"
1692  "enhanced" | "hdf5" | "netCDF-4"
1693  "enhanced-nc3" | "hdf5-nc3" | "netCDF-4 classic model"
1694  */
1695  {
1696  struct Kvalues* kvalue;
1697  char *kind_name = (char *) emalloc(strlen(optarg)+1);
1698  (void)strcpy(kind_name, optarg);
1699  for(kvalue=legalkinds;kvalue->name;kvalue++) {
1700  if(strcmp(kind_name,kvalue->name) == 0) {
1701  option_kind = kvalue->kind;
1702  break;
1703  }
1704  }
1705  if(kvalue->name == NULL) {
1706  error("invalid format: %s", kind_name);
1707  }
1708  }
1709  break;
1710  case 'd': /* non-default compression level specified */
1711  option_deflate_level = strtol(optarg, NULL, 10);
1712  if(option_deflate_level < 0 || option_deflate_level > 9) {
1713  error("invalid deflation level: %d", option_deflate_level);
1714  }
1715  break;
1716  case 's': /* shuffling, may improve compression */
1717  option_shuffle_vars = NC_SHUFFLE;
1718  break;
1719  case 'u': /* convert unlimited dimensions to fixed size */
1720  option_fix_unlimdims = 1;
1721  break;
1722  case 'm': /* non-default size of data copy buffer */
1723  {
1724  double dval;
1725  char *suffix = 0; /* "K" for kilobytes. "M" for megabytes, ... */
1726  dval = strtod(optarg, &suffix);
1727  if(*suffix) {
1728  switch (*suffix) {
1729  case 'k': case 'K':
1730  dval *= 1000;
1731  break;
1732  case 'm': case 'M':
1733  dval *= 1000000;
1734  break;
1735  case 'g': case 'G':
1736  dval *= 1000000000;
1737  break;
1738  case 't': case 'T':
1739  dval *= 1.0e12;
1740  break;
1741  default:
1742  error("If suffix used for '-m' option value, it must be K, M, G, or T: %c",
1743  *suffix);
1744  }
1745  }
1746  option_copy_buffer_size = dval;
1747  break;
1748  }
1749  case 'h': /* non-default size of chunk cache */
1750  {
1751  double dval;
1752  char *suffix = 0; /* "K" for kilobytes, "M" for megabytes, ... */
1753  dval = strtod(optarg, &suffix);
1754  if(*suffix) {
1755  switch (*suffix) {
1756  case 'k': case 'K':
1757  dval *= 1000;
1758  break;
1759  case 'm': case 'M':
1760  dval *= 1000000;
1761  break;
1762  case 'g': case 'G':
1763  dval *= 1000000000;
1764  break;
1765  case 't': case 'T':
1766  dval *= 1.0e12;
1767  break;
1768  default:
1769  error("If suffix used for '-h' option value, it must be K, M, G, or T: %c",
1770  *suffix);
1771  }
1772  }
1773  option_chunk_cache_size = dval;
1774  break;
1775  }
1776  case 'e': /* number of elements chunk cache can hold */
1777  option_chunk_cache_nelems = strtol(optarg, NULL, 10);
1778  if(option_chunk_cache_nelems <= 0) {
1779  error("invalid value for number of chunk cache elements: %d", option_chunk_cache_nelems);
1780  }
1781  break;
1782  case 'r':
1783  option_read_diskless = 1; /* read into memory on open */
1784  break;
1785  case 'w':
1786  option_write_diskless = 1; /* write to memory, persist on close */
1787  break;
1788  case 'x': /* use experimental variable-specific chunk caches */
1789  option_compute_chunkcaches = 1;
1790  break;
1791  case 'c': /* optional chunking spec for each dimension in list */
1792  {
1793  /* save chunkspec string for parsing later, once we know input ncid */
1794  option_chunkspec = strdup(optarg);
1795  break;
1796  }
1797  default:
1798  usage();
1799  }
1800  }
1801  argc -= optind;
1802  argv += optind;
1803 
1804  if (argc != 2) {
1805  error("one input file and one output file required");
1806  }
1807  inputfile = argv[0];
1808  outputfile = argv[1];
1809 
1810  if(strcmp(inputfile, outputfile) == 0) {
1811  error("output would overwrite input");
1812  }
1813 
1814  if(copy(inputfile, outputfile) != NC_NOERR)
1815  exit(1);
1816  return 0;
1817 }
1818 END_OF_MAIN();

Generated on Wed Aug 22 2012 14:39:31 for netCDF. NetCDF is a Unidata library.