diff --git a/src/icartt/dataset.py b/src/icartt/dataset.py index 41d47faf33c1b98766364a9e4c3ce6c5e05797f0..b00c338b16c52a58ec56c05dca645a45955b4a3e 100644 --- a/src/icartt/dataset.py +++ b/src/icartt/dataset.py @@ -25,9 +25,10 @@ class VariableType(IntEnum): DependentVariable = 4 -def sanitize(val, miss): - # TODO: should not use == for float comparison instead, use np.isclose - return float(val) if not float(val) == float(miss) else np.NaN +def vmiss_to_npnan(val, miss): + """converts value to np.nan if is (almost) equal to miss""" + val, miss = map(float, (val, miss)) + return np.NaN if np.isclose(val, miss) else val class DataStore1001: @@ -63,12 +64,12 @@ class DataStore1001: if not self.ivarname in kwargs.keys(): raise Exception("Need independent variable data.") - ivarvalue = sanitize(kwargs[self.ivarname], self.missvals[self.ivarname]) + ivarvalue = vmiss_to_npnan(kwargs[self.ivarname], self.missvals[self.ivarname]) newline = np.array(np.NaN, dtype=[(v, "f8") for v in self.varnames]) for key in kwargs.keys(): if key in self.varnames: - newline[key] = sanitize(kwargs[key], self.missvals[key]) + newline[key] = vmiss_to_npnan(kwargs[key], self.missvals[key]) if type(self.data) is type(None): # TODO: None is the sole instance of None ;-) => can use "if self.data is None" self.data = newline @@ -141,7 +142,7 @@ class DataStore2110(collections.UserDict): def _addBulk(self, raw, n): cur = 0 while cur < n: - ivarvalue = sanitize(raw[cur][0], self.missvals[self.ivarname]) + ivarvalue = vmiss_to_npnan(raw[cur][0], self.missvals[self.ivarname]) self._addAuxline(raw[cur]) cur += 1 @@ -159,7 +160,7 @@ class DataStore2110(collections.UserDict): if not self.ivarname in kwargs.keys(): raise Exception("Need independent variable data.") - ivarvalue = sanitize(kwargs[self.ivarname], self.missvals[self.ivarname]) + ivarvalue = vmiss_to_npnan(kwargs[self.ivarname], self.missvals[self.ivarname]) # this is an AUX line if any([x in self.auxvarnames for x in kwargs.keys()]): # TODO: any (all as well) do not require a comprehension; pure generator expression works fine