NetCDF  4.9.0
dfilter.c
Go to the documentation of this file.
1 /*
2  * Copyright 2018, University Corporation for Atmospheric Research
3  * See netcdf/COPYRIGHT file for copying and redistribution conditions.
4  */
10 #include "config.h"
11 #include <stdlib.h>
12 #include <stdio.h>
13 #include <string.h>
14 #ifdef _MSC_VER
15 #include <io.h>
16 #endif
17 
18 #include "netcdf.h"
19 #include "netcdf_filter.h"
20 #include "ncdispatch.h"
21 #include "nc4internal.h"
22 
23 #ifdef USE_HDF5
24 #include "hdf5internal.h"
25 #endif
26 
27 #ifdef ENABLE_NCZARR
28 #include "zdispatch.h"
29 #endif
30 
31 static void byteswap8(unsigned char*);
32 static void byteswap4(unsigned char*);
33 
34 /*
35 Unified filter related code
36 */
37 
38 /**************************************************/
39 /* Per-variable filters */
40 
60 EXTERNL int
61 nc_inq_var_filter_ids(int ncid, int varid, size_t* nfiltersp, unsigned int* ids)
62 {
63  NC* ncp;
64  int stat = NC_check_id(ncid,&ncp);
65  if(stat != NC_NOERR) return stat;
66  TRACE(nc_inq_var_filter_ids);
67  if((stat = ncp->dispatch->inq_var_filter_ids(ncid,varid,nfiltersp,ids))) goto done;
68 
69 done:
70  return stat;
71 }
72 
96 EXTERNL int
97 nc_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparamsp, unsigned int* params)
98 {
99  NC* ncp;
100  int stat = NC_check_id(ncid,&ncp);
101  if(stat != NC_NOERR) return stat;
102  TRACE(nc_inq_var_filter_info);
103  if((stat = ncp->dispatch->inq_var_filter_info(ncid,varid,id,nparamsp,params))) goto done;
104 
105 done:
106  return stat;
107 }
108 
126 EXTERNL int
127 nc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int* params)
128 {
129  int stat = NC_NOERR;
130  NC* ncp;
131  int fixedsize;
132  nc_type xtype;
133 
134  TRACE(nc_inq_var_filter);
135  if((stat = NC_check_id(ncid,&ncp))) return stat;
136  /* Get variable' type */
137  if((stat = nc_inq_vartype(ncid,varid,&xtype))) return stat;
138  /* If the variable's type is not fixed-size, then signal error */
139  if((stat = NC4_inq_type_fixed_size(ncid, xtype, &fixedsize))) return stat;
140  if(!fixedsize) return NC_EFILTER;
141  if((stat = ncp->dispatch->def_var_filter(ncid,varid,id,nparams,params))) goto done;
142 done:
143  return stat;
144 }
145 
173 EXTERNL int
174 nc_inq_var_filter(int ncid, int varid, unsigned int* idp, size_t* nparamsp, unsigned int* params)
175 {
176  NC* ncp;
177  size_t nfilters;
178  unsigned int* ids = NULL;
179  int stat = NC_check_id(ncid,&ncp);
180 
181  if(stat != NC_NOERR) return stat;
182  TRACE(nc_inq_var_filter);
183 
184  /* Get the number of filters on this variable */
185  if((stat = nc_inq_var_filter_ids(ncid,varid,&nfilters, NULL))) goto done;
186  /* If no filters, then return zero */
187  if(nfilters == 0) {
188  if(idp) *idp = 0;
189  goto done;
190  }
191  /* Get the filter ids */
192  if((ids = calloc(sizeof(unsigned int),nfilters)) == NULL) {stat = NC_ENOMEM; goto done;}
193  if((stat = nc_inq_var_filter_ids(ncid,varid,&nfilters, ids))) goto done;
194  /* Get params for the first filter */
195  if((stat = nc_inq_var_filter_info(ncid,varid,ids[0],nparamsp,params))) goto done;
196  if(idp) *idp = ids[0];
197  done:
198  nullfree(ids);
199  return stat;
200 }
201 
215 EXTERNL int
216 nc_inq_filter_avail(int ncid, unsigned id)
217 {
218  int stat = NC_NOERR;
219  NC* ncp;
220 
221  stat = NC_check_id(ncid,&ncp);
222  if(stat != NC_NOERR) return stat;
223  if((stat = ncp->dispatch->inq_filter_avail(ncid,id))) goto done;
224 done:
225  return stat;
226 }
227 
228 /**************************************************/
229 /* Support direct user defined filters */
230 
231 #ifdef ENABLE_CLIENTSIDE_FILTERS
232 
244 EXTERNL int
245 nc_filter_client_register(unsigned int id, void* info)
246 {
247  int stat = NC_NOERR;
248 #ifdef USE_HDF5
249  NC_FILTER_OBJ_HDF5 client;
250  if(id == 0 ||info == NULL)
251  return NC_EINVAL;
252  memset(&client,0,sizeof(client));
253  client.hdr.format = NC_FILTER_FORMAT_HDF5;
254  client.sort = NC_FILTER_SORT_CLIENT;
255  client.u.client.id = id;
256  client.u.client.info = info;
257  /* Note use of a global function, not part of the dispatch table */
258  stat = nc4_global_filter_action(NCFILTER_CLIENT_REG, id, &client);
259 #else
260  stat = NC_ENOTBUILT;
261 #endif
262  return stat;
263 }
264 
273 EXTERNL int
274 nc_filter_client_unregister(unsigned int id)
275 {
276 int stat = NC_NOERR;
277 #ifdef USE_HDF5
278  stat = nc4_global_filter_action(NCFILTER_CLIENT_UNREG, id, NULL);
279 #else
280  stat = NC_ENOTBUILT;
281 #endif
282  return stat;
283 }
284 
294 EXTERNL int
295 nc_filter_client_inq(unsigned int id, void* infop)
296 {
297 int stat = NC_NOERR;
298 #ifdef USE_HDF5
299  H5Z_class2_t* hct = (H5Z_class2_t*)infop;
300  NC_FILTER_OBJ_HDF5 client;
301  if(id == 0 ||infop == NULL)
302  return NC_EINVAL;
303  memset(&client,0,sizeof(client));
304  client.hdr.format = NC_FILTER_FORMAT_HDF5;
305  client.sort = NC_FILTER_SORT_CLIENT;
306  client.u.client.id = id;
307  client.u.client.info = hct;
308  /* Note use of a global function, not part of the dispatch table */
309  stat = nc4_global_filter_action(NCFILTER_CLIENT_INQ, id, &client);
310  if(stat == NC_NOERR) {
311  *hct = *(H5Z_class2_t*)client.u.client.info;
312  }
313 #else
314  stat = NC_ENOTBUILT;
315 #endif
316  return stat;
317 }
318 #endif /*ENABLE_CLIENTSIDE_FILTERS*/
319 
320 /**************************************************/
321 /* Functions for accessing standardized filters */
322 
334 int
335 nc_def_var_bzip2(int ncid, int varid, int level)
336 {
337  int stat = NC_NOERR;
338  unsigned ulevel;
339 
340  if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_BZIP2))) goto done;
341  /* Filter is available */
342  /* 1 <= Level <= 9 */
343  if (level < 1 || level > 9)
344  return NC_EINVAL;
345  ulevel = (unsigned) level; /* Keep bit pattern */
346  if((stat = nc_def_var_filter(ncid,varid,H5Z_FILTER_BZIP2,1,&ulevel))) goto done;
347 done:
348  return stat;
349 }
350 
365 int
366 nc_inq_var_bzip2(int ncid, int varid, int* hasfilterp, int *levelp)
367 {
368  int stat = NC_NOERR;
369  size_t nparams;
370  unsigned params = 0;
371  int hasfilter = 0;
372 
373  if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_BZIP2))) goto done;
374  /* Filter is available */
375  /* Get filter info */
376  stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_BZIP2,&nparams,NULL);
377  if(stat == NC_ENOFILTER) {stat = NC_NOERR; hasfilter = 0; goto done;}
378  if(stat != NC_NOERR) goto done;
379  hasfilter = 1;
380  if(nparams != 1) {stat = NC_EFILTER; goto done;}
381  if((stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_BZIP2,&nparams,&params))) goto done;
382 done:
383  if(levelp) *levelp = (int)params;
384  if(hasfilterp) *hasfilterp = hasfilter;
385  return stat;
386 }
387 
402 int
403 nc_def_var_zstandard(int ncid, int varid, int level)
404 {
405 #ifdef HAVE_ZSTD
406  int stat = NC_NOERR;
407  unsigned ulevel;
408 
409  if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_ZSTD))) goto done;
410  /* Filter is available */
411  /* Level must be between -131072 and 22 on Zstandard v. 1.4.5 (~202009)
412  Earlier versions have fewer levels (especially fewer negative levels) */
413  if (level < -131072 || level > 22)
414  return NC_EINVAL;
415  ulevel = (unsigned) level; /* Keep bit pattern */
416  if((stat = nc_def_var_filter(ncid,varid,H5Z_FILTER_ZSTD,1,&ulevel))) goto done;
417 done:
418  return stat;
419 #else
420  return NC_NOERR;
421 #endif /*HAVE_ZSTD*/
422 }
423 
438 int
439 nc_inq_var_zstandard(int ncid, int varid, int* hasfilterp, int *levelp)
440 {
441 #ifdef HAVE_ZSTD
442  int stat = NC_NOERR;
443  size_t nparams;
444  unsigned params = 0;
445  int hasfilter = 0;
446 
447  if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_ZSTD))) goto done;
448  /* Filter is available */
449  /* Get filter info */
450  stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_ZSTD,&nparams,NULL);
451  if(stat == NC_ENOFILTER) {stat = NC_NOERR; hasfilter = 0; goto done;}
452  if(stat != NC_NOERR) goto done;
453  hasfilter = 1;
454  if(nparams != 1) {stat = NC_EFILTER; goto done;}
455  if((stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_ZSTD,&nparams,&params))) goto done;
456 done:
457  if(levelp) *levelp = (int)params;
458  if(hasfilterp) *hasfilterp = hasfilter;
459  return stat;
460 #else
461  return NC_NOERR;
462 #endif /*HAVE_ZSTD*/
463 }
464 
479 int
480 nc_def_var_blosc(int ncid, int varid, unsigned subcompressor, unsigned level, unsigned blocksize, unsigned addshuffle)
481 {
482 #ifdef HAVE_BLOSC
483  int stat = NC_NOERR;
484  unsigned params[7];;
485 
486  if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_BLOSC))) goto done;
487  /* Filter is available */
488 
489  /* Verify parameters */
490  if(addshuffle > (unsigned)BLOSC_BITSHUFFLE) {stat = NC_EINVAL; goto done;}
491  if(subcompressor > (unsigned)BLOSC_ZSTD) {stat = NC_EINVAL; goto done;}
492 
493  /* Set the parameters */
494  params[0] = 0;
495  params[1] = 0;
496  params[2] = 0;
497  params[3] = blocksize;
498  params[4] = level;
499  params[5] = addshuffle;
500  params[6] = subcompressor;
501  if((stat = nc_def_var_filter(ncid,varid,H5Z_FILTER_BLOSC,7,params))) goto done;
502 done:
503  return stat;
504 #else
505  return NC_NOERR;
506 #endif
507 }
508 
529 int
530 nc_inq_var_blosc(int ncid, int varid, int* hasfilterp, unsigned* subcompressorp,
531  unsigned* levelp, unsigned* blocksizep, unsigned* addshufflep)
532 {
533 #ifdef HAVE_BLOSC
534  int stat = NC_NOERR;
535  size_t nparams;
536  unsigned params[7];
537  int hasfilter = 0;
538 
539  if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_BLOSC))) goto done;
540  /* Filter is available */
541 
542  /* Get filter info */
543  stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_BLOSC,&nparams,NULL);
544  if(stat == NC_ENOFILTER) {stat = NC_NOERR; hasfilter = 0; goto done;}
545  if(stat != NC_NOERR) goto done;
546  hasfilter = 1;
547  if(nparams != 7) {stat = NC_EFILTER; goto done;}
548  if((stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_BLOSC,&nparams,params))) goto done;
549  if(blocksizep) *blocksizep = params[3];
550  if(levelp) *levelp = params[4];
551  if(addshufflep) *addshufflep = params[5];
552  if(subcompressorp) *subcompressorp = params[6];
553 done:
554  if(hasfilterp) *hasfilterp = hasfilter;
555  return stat;
556 #else
557  return NC_NOERR;
558 #endif
559 }
int nc_inq_var_blosc(int ncid, int varid, int *hasfilterp, unsigned *subcompressorp, unsigned *levelp, unsigned *blocksizep, unsigned *addshufflep)
Learn whether Blosc compression is on for a variable, and, if so, the settings.
Definition: dfilter.c:530
int nc_def_var_zstandard(int ncid, int varid, int level)
Turn on Zstandard compression for a variable.
Definition: dfilter.c:403
int nc_inq_var_bzip2(int ncid, int varid, int *hasfilterp, int *levelp)
Learn whether bzip2 compression is on for a variable, and, if so, the level setting.
Definition: dfilter.c:366
int nc_def_var_bzip2(int ncid, int varid, int level)
Turn on bzip2 compression for a variable.
Definition: dfilter.c:335
EXTERNL int nc_inq_filter_avail(int ncid, unsigned id)
Test if filter is available.
Definition: dfilter.c:216
int nc_inq_var_zstandard(int ncid, int varid, int *hasfilterp, int *levelp)
Learn whether Zstandard compression is on for a variable, and, if so, the level setting.
Definition: dfilter.c:439
EXTERNL int nc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int *params)
Define a new variable filter Assumes HDF5 format using unsigned ints.
Definition: dfilter.c:127
EXTERNL int nc_inq_vartype(int ncid, int varid, nc_type *xtypep)
Learn the type of a variable.
Definition: dvarinq.c:178
int nc_def_var_blosc(int ncid, int varid, unsigned subcompressor, unsigned level, unsigned blocksize, unsigned addshuffle)
Turn on blosc for a variable.
Definition: dfilter.c:480
EXTERNL int nc_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t *nparamsp, unsigned int *params)
Find the the param info about filter (if any) associated with a variable and with specified id.
Definition: dfilter.c:97
EXTERNL int nc_inq_var_filter(int ncid, int varid, unsigned int *idp, size_t *nparamsp, unsigned int *params)
Find the first filter (if any) associated with a variable.
Definition: dfilter.c:174
EXTERNL int nc_inq_var_filter_ids(int ncid, int varid, size_t *nfiltersp, unsigned int *ids)
Find the set of filters (if any) associated with a variable.
Definition: dfilter.c:61
Main header file for the C API.
#define NC_EFILTER
Filter operation failed.
Definition: netcdf.h:513
#define NC_ENOMEM
Memory allocation (malloc) failure.
Definition: netcdf.h:448
#define EXTERNL
Needed for DLL build.
Definition: netcdf.h:556
#define NC_EINVAL
Invalid Argument.
Definition: netcdf.h:378
#define NC_NOERR
No Error.
Definition: netcdf.h:368
#define NC_ENOTBUILT
Attempt to use feature that was not turned on when netCDF was built.
Definition: netcdf.h:508
#define NC_ENOFILTER
Filter not defined on variable.
Definition: netcdf.h:517
int nc_type
The nc_type type is just an int.
Definition: netcdf.h:25