Merge remote-tracking branch 'mbees-public/master'

docs/index.rst

@@ -37,6 +37,17 @@ Dataset
 .. autoclass:: Dataset
    :members:
 
+DataStore
+*********
+
+The .data attribute of an ``icartt.Dataset`` is a DataStore, which can be accessed to add data as follows:
+
+.. autoclass:: DataStore1001
+   :members:
+
+.. autoclass:: DataStore2110
+   :members:
+
 Formats
 ********
 

src/icartt/__init__.py

@@ -17,6 +17,22 @@ del get_version
 
 
 # EXPORTED TYPES
-from .dataset import Dataset, StandardNormalComments, Variable, Formats, VariableType
+from .dataset import (
+    Dataset,
+    StandardNormalComments,
+    Variable,
+    Formats,
+    VariableType,
+    DataStore1001,
+    DataStore2110,
+)
 
-__all__ = ("Dataset", "StandardNormalComments", "Variable", "Formats", "VariableType")
+__all__ = (
+    "Dataset",
+    "StandardNormalComments",
+    "Variable",
+    "Formats",
+    "VariableType",
+    "DataStore1001",
+    "DataStore2110",
+)

src/icartt/dataset.py

@@ -30,6 +30,8 @@ class VariableType(IntEnum):
 
 
 class DataStore1001:
+    """Data model for FFI1001"""
+
     def __init__(self, ivar, dvars):
         self.ivarname = ivar.shortname
 
@@ -111,6 +113,8 @@ class DataStore1001:
 
 
 class DataStore2110(collections.UserDict):
+    """Data model for FFI2110"""
+
     def __init__(self, ivar, ibvar, auxvars, dvars):
         self.ivarname = ivar.shortname
         self.ibvarname = ibvar.shortname
@@ -162,6 +166,19 @@ class DataStore2110(collections.UserDict):
             self.data[ivarValue] = {"AUX": auxds, "DEP": depds}
 
     def add(self, newAuxData, newDepData):
+        """(bulk) add data, providing (structured) numpy arrays for both the auxiliary and dependent data line(s)
+        for a given ivar value.
+
+        Arrays have to have shape [ (ivar, auxvar, auxvar, ...) ] and
+        [ (ibvar, depvar, depvar, ...), ... ] for auxiliary and dependent data line(s), respectively.
+        missing values have to be set to np.nan.
+
+        :param newAuxData: auxiliary data line to be added
+        :type newAuxData: numpy.ndarray
+
+        :param newDepData: auxiliary data line(s) to be added
+        :type newDepData: numpy.ndarray
+        """
         auxds = DataStore1001(self.ivar, self.auxvars)
         depds = DataStore1001(self.ibvar, self.dvars)
 
@@ -424,7 +441,12 @@ class Dataset:
         if self.defineMode:
             return np.datetime64("NaT")
 
-        if self.data.data is None or self.data.data == {}:
+        # for 1001, its an array, for 2110 a dict
+        if not isinstance(self.data.data, (np.ndarray, dict)):
+            return np.datetime64("NaT")
+
+        # it is possible to have an empty dict
+        if isinstance(self.data.data, dict) and not self.data.data:
             return np.datetime64("NaT")
 
         ref_dt = np.datetime64(datetime.datetime(*self.dateOfCollection), "ns")

tests/.gitignore

-/.coverage
+/.coverage
\ No newline at end of file

tests/usage_examples/create_ffi1001.py

@@ -2,6 +2,9 @@ import datetime
 
 import icartt
 
+# ------------------------------------------------------
+# Phase 1: define file format, properties and variables
+# ------------------------------------------------------
 
 ict = icartt.Dataset(format=icartt.Formats.FFI1001)
 
@@ -48,19 +51,28 @@ ict.normalComments.freeform.append("free comment line 2")
 
 ict.endDefineMode()
 
-# Add data
+# ------------------------------------------------------
+# Phase 2: after ending define mode, add data
+# ------------------------------------------------------
+
+# all data are stored as NumPy arrays, and need to be provided as such.
 
 import numpy as np
 
-data = np.array([(13.4, 14.0, 2348925e5), (14.1, 14.9, 23425634e5)])
+#                 ivar  dvar1 dvar2
+data = np.array([(15.4, 15.0, 52452495290e5)])
 ict.data.add(data)
 
 # Note 1: you are responsible to ensure that the order of elements in a data line
 # corresponds to variable listing below:
 print([x for x in ict.variables])
 
-# Note 2: for single lines, you still need to make it an array!
-data = np.array([(15.4, 15.0, 52452495290e5)])
+# Note 2: evenfor single lines, you still need to make it an array!
+
+# Note 3: multiple data lines can be added in bulk!
+#                 ivar  dvar1 dvar2       ivar  dvar1 dvar2      ...
+data = np.array([(13.4, 14.0, 2348925e5), (14.1, 14.9, 23425634e5)])
+
 ict.data.add(data)
 
 # Now, look at it in ICARTT form:

tests/usage_examples/create_ffi2110.py

@@ -4,6 +4,9 @@ import numpy as np
 
 import icartt
 
+# ------------------------------------------------------
+# Phase 1: define file format, properties and variables
+# ------------------------------------------------------
 
 ict = icartt.Dataset(format=icartt.Formats.FFI2110)
 
@@ -82,25 +85,27 @@ ict.specialComments.append("Adapt as needed.")
 
 ict.endDefineMode()
 
-# Add data
-
-# for 2110, data can be added as chunks for a given ivar!
+# ------------------------------------------------------
+# Phase 2: after ending define mode, add data
+# ------------------------------------------------------
 
 import numpy as np
 
+# same as for 1001, but data needs to be added in chunks for each ivar value
+
 # note, the second variable ('4') is the number of dependent lines to follow
-#                       ivar, ndepvar, auxvar1, auxvar2, auxvar3
+#                   ivar, ndepvar, auxvar1, auxvar2, auxvar3
 auxData = np.array([(12.3, 4, 12.5, 48.21, 10.3)])
-#                   ibvar,  dvar1,  dvar2
+#                   ibvar,  dvar1,  dvar2  ... (repeat ndepvar times)
 depData = np.array(
     [(0, 123, 8.4e4), (100, 122, 9.1e4), (250, 115, 9.3e4), (500, 106, 9.8e4)]
 )
 
 ict.data.add(auxData, depData)
 
-# ... and so forth
-auxData = np.array([(12.4, 2, 12.8, 48.41, 12.1)])
+# ... and so forth for the next chunk:
 
+auxData = np.array([(12.4, 2, 12.8, 48.41, 12.1)])
 #                   ibvar,  dvar1,  dvar2
 depData = np.array([(0, 153, 7.3e4), (270, 172, 8.9e4)])
 

tests/usage_examples/read_ffi2110.py

@@ -11,29 +11,26 @@ ict = icartt.Dataset(
 [x for x in ict.variables]
 
 # independent, independent bounded, dependent, auxiliary variables?
-print(f"Independent variable: {ict.independentVariable.shortname}")
-print(f"Independent bounded variable: {ict.independentBoundedVariable.shortname}")
-print(f"Auxiliary variables: {', '.join([ x for x in ict.auxiliaryVariables])}")
-print(f"Dependent variables: {', '.join([ x for x in ict.dependentVariables])}")
+ict.independentVariable.shortname
+ict.independentBoundedVariable.shortname
+", ".join([x for x in ict.auxiliaryVariables])
+", ".join([x for x in ict.dependentVariables])
 
-# some info on a variable
-print(f"Units of variable Latitude are {ict.variables['Latitude'].units}")
-print(f"... and its missing value is {ict.variables['Latitude'].miss}")
+# some info on a variable (units, missing value of "Latitude")
+ict.variables["Latitude"].units
+ict.variables["Latitude"].miss
 
 # get steps for which data is available:
 tsteps = [x for x in ict.data]
 
 # let's look at the first time step data
-print("First time step data:")
-print(ict.data[tsteps[0]])
+ict.data[tsteps[0]]
 
 # auxiliary data at this time step:
-print("First time step auxiliary data:")
-print(ict.data[tsteps[0]]["AUX"][:])
+ict.data[tsteps[0]]["AUX"][:]
 
 # dependent data at this time step:
 tstepdata = ict.data[tsteps[0]]["DEP"][:]
 
 # get the ozone mixing ratio for those data where Altitude < 10000.0:
-print(f"Ozone mixing ratio for altitudes < 10000 at time step {tsteps[0]}")
-print(tstepdata[tstepdata["Altitude[]"] < 10000.0]["O3_MR[]"])
+tstepdata[tstepdata["Altitude[]"] < 10000.0]["O3_MR[]"]