From 1b1623832afdeca2e544359148d2249788d5373b Mon Sep 17 00:00:00 2001 From: Christoph Knote Date: Mon, 28 Mar 2022 18:00:45 +0200 Subject: [PATCH 1/2] Add docs, tests --- .gitignore | 2 ++ docs/index.rst | 11 +++++++++++ src/icartt/__init__.py | 20 ++++++++++++++++++-- src/icartt/dataset.py | 24 +++++++++++++++++++++++- tests/.gitignore | 2 +- tests/usage_examples/create_ffi1001.py | 20 ++++++++++++++++---- tests/usage_examples/create_ffi2110.py | 19 ++++++++++++------- tests/usage_examples/read_ffi2110.py | 23 ++++++++++------------- 8 files changed, 93 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index f0195f9..ba3a7c0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +public/ + # Mac **.DS_Store diff --git a/docs/index.rst b/docs/index.rst index 7c62f55..88055f5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -37,6 +37,17 @@ Dataset .. autoclass:: Dataset :members: +DataStore +********* + +The .data attribute of an ``icartt.Dataset`` is a DataStore, which can be accessed to add data as follows: + +.. autoclass:: DataStore1001 + :members: + +.. autoclass:: DataStore2110 + :members: + Formats ******** diff --git a/src/icartt/__init__.py b/src/icartt/__init__.py index 1e51075..dd59964 100644 --- a/src/icartt/__init__.py +++ b/src/icartt/__init__.py @@ -17,6 +17,22 @@ del get_version # EXPORTED TYPES -from .dataset import Dataset, StandardNormalComments, Variable, Formats, VariableType +from .dataset import ( + Dataset, + StandardNormalComments, + Variable, + Formats, + VariableType, + DataStore1001, + DataStore2110, +) -__all__ = ("Dataset", "StandardNormalComments", "Variable", "Formats", "VariableType") +__all__ = ( + "Dataset", + "StandardNormalComments", + "Variable", + "Formats", + "VariableType", + "DataStore1001", + "DataStore2110", +) diff --git a/src/icartt/dataset.py b/src/icartt/dataset.py index b8abad9..1712616 100644 --- a/src/icartt/dataset.py +++ b/src/icartt/dataset.py @@ -30,6 +30,8 @@ class VariableType(IntEnum): class DataStore1001: + """Data model for FFI1001""" + def __init__(self, ivar, dvars): self.ivarname = ivar.shortname @@ -111,6 +113,8 @@ class DataStore1001: class DataStore2110(collections.UserDict): + """Data model for FFI1001""" + def __init__(self, ivar, ibvar, auxvars, dvars): self.ivarname = ivar.shortname self.ibvarname = ibvar.shortname @@ -162,6 +166,19 @@ class DataStore2110(collections.UserDict): self.data[ivarValue] = {"AUX": auxds, "DEP": depds} def add(self, newAuxData, newDepData): + """(bulk) add data, providing (structured) numpy arrays for both the auxiliary and dependent data line(s) + for a given ivar value. + + Arrays have to have shape [ (ivar, auxvar, auxvar, ...) ] and + [ (ibvar, depvar, depvar, ...), ... ] for auxiliary and dependent data line(s), respectively. + missing values have to be set to np.nan. + + :param newAuxData: auxiliary data line to be added + :type newAuxData: numpy.ndarray + + :param newDepData: auxiliary data line(s) to be added + :type newDepData: numpy.ndarray + """ auxds = DataStore1001(self.ivar, self.auxvars) depds = DataStore1001(self.ibvar, self.dvars) @@ -424,7 +441,12 @@ class Dataset: if self.defineMode: return np.datetime64("NaT") - if self.data.data is None or self.data.data == {}: + # for 1001, its an array, for 2110 a dict + if not isinstance(self.data.data, (np.ndarray, dict)): + return np.datetime64("NaT") + + # it is possible to have an empty dict + if isinstance(self.data.data, dict) and not self.data.data: return np.datetime64("NaT") ref_dt = np.datetime64(datetime.datetime(*self.dateOfCollection), "ns") diff --git a/tests/.gitignore b/tests/.gitignore index 1bc5aa1..ca7ad6b 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1 +1 @@ -/.coverage +/.coverage \ No newline at end of file diff --git a/tests/usage_examples/create_ffi1001.py b/tests/usage_examples/create_ffi1001.py index 15602ee..b544fe7 100644 --- a/tests/usage_examples/create_ffi1001.py +++ b/tests/usage_examples/create_ffi1001.py @@ -2,6 +2,9 @@ import datetime import icartt +# ------------------------------------------------------ +# Phase 1: define file format, properties and variables +# ------------------------------------------------------ ict = icartt.Dataset(format=icartt.Formats.FFI1001) @@ -48,19 +51,28 @@ ict.normalComments.freeform.append("free comment line 2") ict.endDefineMode() -# Add data +# ------------------------------------------------------ +# Phase 2: after ending define mode, add data +# ------------------------------------------------------ + +# all data are stored as NumPy arrays, and need to be provided as such. import numpy as np -data = np.array([(13.4, 14.0, 2348925e5), (14.1, 14.9, 23425634e5)]) +# ivar dvar1 dvar2 +data = np.array([(15.4, 15.0, 52452495290e5)]) ict.data.add(data) # Note 1: you are responsible to ensure that the order of elements in a data line # corresponds to variable listing below: print([x for x in ict.variables]) -# Note 2: for single lines, you still need to make it an array! -data = np.array([(15.4, 15.0, 52452495290e5)]) +# Note 2: evenfor single lines, you still need to make it an array! + +# Note 3: multiple data lines can be added in bulk! +# ivar dvar1 dvar2 ivar dvar1 dvar2 ... +data = np.array([(13.4, 14.0, 2348925e5), (14.1, 14.9, 23425634e5)]) + ict.data.add(data) # Now, look at it in ICARTT form: diff --git a/tests/usage_examples/create_ffi2110.py b/tests/usage_examples/create_ffi2110.py index aee2dfb..feadd89 100644 --- a/tests/usage_examples/create_ffi2110.py +++ b/tests/usage_examples/create_ffi2110.py @@ -4,6 +4,9 @@ import numpy as np import icartt +# ------------------------------------------------------ +# Phase 1: define file format, properties and variables +# ------------------------------------------------------ ict = icartt.Dataset(format=icartt.Formats.FFI2110) @@ -82,25 +85,27 @@ ict.specialComments.append("Adapt as needed.") ict.endDefineMode() -# Add data - -# for 2110, data can be added as chunks for a given ivar! +# ------------------------------------------------------ +# Phase 2: after ending define mode, add data +# ------------------------------------------------------ import numpy as np +# same as for 1001, but data needs to be added in chunks for each ivar value + # note, the second variable ('4') is the number of dependent lines to follow -# ivar, ndepvar, auxvar1, auxvar2, auxvar3 +# ivar, ndepvar, auxvar1, auxvar2, auxvar3 auxData = np.array([(12.3, 4, 12.5, 48.21, 10.3)]) -# ibvar, dvar1, dvar2 +# ibvar, dvar1, dvar2 ... (repeat ndepvar times) depData = np.array( [(0, 123, 8.4e4), (100, 122, 9.1e4), (250, 115, 9.3e4), (500, 106, 9.8e4)] ) ict.data.add(auxData, depData) -# ... and so forth -auxData = np.array([(12.4, 2, 12.8, 48.41, 12.1)]) +# ... and so forth for the next chunk: +auxData = np.array([(12.4, 2, 12.8, 48.41, 12.1)]) # ibvar, dvar1, dvar2 depData = np.array([(0, 153, 7.3e4), (270, 172, 8.9e4)]) diff --git a/tests/usage_examples/read_ffi2110.py b/tests/usage_examples/read_ffi2110.py index 4189aaf..946a540 100644 --- a/tests/usage_examples/read_ffi2110.py +++ b/tests/usage_examples/read_ffi2110.py @@ -11,29 +11,26 @@ ict = icartt.Dataset( [x for x in ict.variables] # independent, independent bounded, dependent, auxiliary variables? -print(f"Independent variable: {ict.independentVariable.shortname}") -print(f"Independent bounded variable: {ict.independentBoundedVariable.shortname}") -print(f"Auxiliary variables: {', '.join([ x for x in ict.auxiliaryVariables])}") -print(f"Dependent variables: {', '.join([ x for x in ict.dependentVariables])}") +ict.independentVariable.shortname +ict.independentBoundedVariable.shortname +", ".join([x for x in ict.auxiliaryVariables]) +", ".join([x for x in ict.dependentVariables]) -# some info on a variable -print(f"Units of variable Latitude are {ict.variables['Latitude'].units}") -print(f"... and its missing value is {ict.variables['Latitude'].miss}") +# some info on a variable (units, missing value of "Latitude") +ict.variables["Latitude"].units +ict.variables["Latitude"].miss # get steps for which data is available: tsteps = [x for x in ict.data] # let's look at the first time step data -print("First time step data:") -print(ict.data[tsteps[0]]) +ict.data[tsteps[0]] # auxiliary data at this time step: -print("First time step auxiliary data:") -print(ict.data[tsteps[0]]["AUX"][:]) +ict.data[tsteps[0]]["AUX"][:] # dependent data at this time step: tstepdata = ict.data[tsteps[0]]["DEP"][:] # get the ozone mixing ratio for those data where Altitude < 10000.0: -print(f"Ozone mixing ratio for altitudes < 10000 at time step {tsteps[0]}") -print(tstepdata[tstepdata["Altitude[]"] < 10000.0]["O3_MR[]"]) +tstepdata[tstepdata["Altitude[]"] < 10000.0]["O3_MR[]"] -- GitLab From be299ce387aad3a6735fb9e6aa19fce95fbb28c7 Mon Sep 17 00:00:00 2001 From: Christoph Knote Date: Tue, 29 Mar 2022 10:36:02 +0200 Subject: [PATCH 2/2] Fix typo --- src/icartt/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/icartt/dataset.py b/src/icartt/dataset.py index 1712616..0e035d1 100644 --- a/src/icartt/dataset.py +++ b/src/icartt/dataset.py @@ -113,7 +113,7 @@ class DataStore1001: class DataStore2110(collections.UserDict): - """Data model for FFI1001""" + """Data model for FFI2110""" def __init__(self, ivar, ibvar, auxvars, dvars): self.ivarname = ivar.shortname -- GitLab