From b93bf20de5b675d52fe33b3f530f70bd0f12d03c Mon Sep 17 00:00:00 2001 From: Florian Obersteiner Date: Thu, 3 Mar 2022 11:01:27 +0100 Subject: [PATCH] .times method --- src/icartt/dataset.py | 50 +++++++++++++++++++------------------------ tests/test_dataset.py | 18 ++++++++++++++-- 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/src/icartt/dataset.py b/src/icartt/dataset.py index 29d7d4c..ffd0a32 100644 --- a/src/icartt/dataset.py +++ b/src/icartt/dataset.py @@ -44,7 +44,10 @@ class DataStore1001: self.data = None def __getitem__(self, s=slice(None)): - return self.data[s] + # we can only slice if we have something, so + if self.data is not None: + return self.data[s] + # returns None implicitly if self.data is None def addBulk(self, raw): nlines, nvars = raw.shape @@ -117,7 +120,10 @@ class DataStore2110(collections.UserDict): self.dvars = dvars def __getitem__(self, s=slice(None)): - return self.data[s] + # we can only slice if we have something, so + if self.data is not None: + return self.data[s] + # returns None implicitly if self.data is None def _addAuxline(self, auxline): newdata = { @@ -421,13 +427,13 @@ class Dataset: :return: list of time steps :rtype: list """ - # TODO: this method currently does not work at all. - # suggest to also change to return a numpy array of numpy.datetime64 - # for consistency with other data output - return [ - self.dateOfCollection + datetime.timedelta(seconds=x) - for x in self.independentVariable - ] + if self.data.data is None or self.independentVariable is None: + return np.datetime64('NaT') + + ref_dt = np.datetime64(datetime.datetime(*self.dateOfCollection), 'ns') + + # ivar unit is seconds as per standard; need to convert to ns to use timedelta64[ns] type. + return ref_dt + (self.data[self.independentVariable.shortname]*10**9).astype('timedelta64[ns]') @property def variables(self): @@ -510,11 +516,8 @@ class Dataset: # line 7 - UTC date when data begin, UTC date of data reduction or revision # - comma delimited (yyyy, mm, dd, yyyy, mm, dd). dmp = f.readline() - self.dateOfCollection = datetime.datetime(*map(int, dmp[:3])) - self.dateOfRevision = datetime.datetime(*map(int, dmp[3:6])) - # TODO: we should either use aware datetime (UTC), date objects or - # numpy.datetime64 here to avoid some 'rough edges' of Python's datetime library... - + self.dateOfCollection = tuple(map(int, dmp[:3])) + self.dateOfRevision = tuple(map(int, dmp[3:6])) # line 8 - Data Interval (This value describes the time spacing (in seconds) # between consecutive data records. It is the (constant) interval between @@ -696,7 +699,7 @@ class Dataset: + "_" + self.locationID + "_" - + datetime.datetime.strftime(self.dateOfCollection, dateFormat) + + datetime.datetime.strftime(datetime.datetime(*self.dateOfCollection), dateFormat) ) fn += "_R" + str(self.revision) if not self.revision is None else "" fn += "_L" + str(self.launch) if not self.launch is None else "" @@ -755,13 +758,7 @@ class Dataset: ) # UTC date when data begin, UTC date of data reduction or revision - comma delimited (yyyy, mm, dd, yyyy, mm, dd). write_to_file( - delimiter.join( - [ - # TODO: if we use anything other than datetime.datetime, we'll have to ensure this still works... - datetime.datetime.strftime(x, delimiter.join(["%Y", "%m", "%d"])) - for x in [self.dateOfCollection, self.dateOfRevision] - ] - ) + delimiter.join(f"{x:02d}" for x in (*self.dateOfCollection, *self.dateOfRevision)) ) # Data Interval (This value describes the time spacing (in seconds) between consecutive data records. It is the (constant) interval between values of the independent variable. For 1 Hz data the data interval value is 1 and for 10 Hz data the value is 0.1. All intervals longer than 1 second must be reported as Start and Stop times, and the Data Interval value is set to 0. The Mid-point time is required when it is not at the average of Start and Stop times. For additional information see Section 2.5 below.). write_to_file(delimiter.join([str(x) for x in self.dataIntervalCode])) @@ -894,12 +891,9 @@ class Dataset: self.dataSourceDescription = "Musterdatenprodukt" self.missionName = "MUSTEREX" - # TODO: see also comment on _readData method. - # we should either use aware datetime (UTC), date objects or - # numpy.datetime64 here to avoid some 'rough edges' of Python's - # datetime library... - self.dateOfCollection = datetime.datetime.today() - self.dateOfRevision = datetime.datetime.today() + # .utcnow() should not be used in general, but it is ok if you just need the timetuple. + self.dateOfCollection = datetime.datetime.utcnow().timetuple()[:3] + self.dateOfRevision = datetime.datetime.utcnow().timetuple()[:3] self.dataIntervalCode = [0.0] self.independentVariable = None diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 2c78a1a..e132a53 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -255,14 +255,16 @@ class Simple1001TestCase(unittest.TestCase): class Create1001TestCase(unittest.TestCase): def testCreateDs(self): + now = datetime.datetime.today() + ict = icartt.Dataset(format=icartt.Formats.FFI1001) ict.PIName = "Knote, Christoph" ict.PIAffiliation = "Faculty of Medicine, University Augsburg, Germany" ict.dataSourceDescription = "Example data" ict.missionName = "MBEES" - ict.dateOfCollection = datetime.datetime.today() - ict.dateOfRevision = datetime.datetime.today() + ict.dateOfCollection = now.timetuple()[:3] + ict.dateOfRevision = now.timetuple()[:3] ict.dataIntervalCode = [0] @@ -295,6 +297,11 @@ class Create1001TestCase(unittest.TestCase): ict.endDefineMode() + # we have not added data yet, so data must be None + self.assertIsNone(ict.data[:]) + # and times must be NaT + self.assertTrue(np.isnat(ict.times)) + ict.data.add(Time_Start=12.3, Time_Stop=12.5, Payload=23789423.2e5) mydict = {"Time_Start": 12.6, "Time_Stop": 13.1, "Payload": 324235644.1e5} @@ -303,6 +310,11 @@ class Create1001TestCase(unittest.TestCase): data = np.array([(13.4, 14.0, 2348925e5), (14.1, 14.9, 23425634e5)]) ict.data.addBulk(data) + # elements of the time array must be equal to our input + t0 = np.datetime64(datetime.datetime(*now.timetuple()[:3]), 'ns') + for have, want in zip(ict.times, (12.3, 12.6, 13.4, 14.1)): + self.assertEqual(int(have-t0), int(want*10**9)) + strOut = io.StringIO() ict.write(f=strOut) @@ -310,6 +322,8 @@ class Create1001TestCase(unittest.TestCase): return True + + class BulkIOTestCase(unittest.TestCase): # TODO: need to test invalid input # def testInvalid(self): -- GitLab