From b3064e6a4f2a981b44646f0174efdeb6c2f4cd90 Mon Sep 17 00:00:00 2001 From: Florian Obersteiner Date: Sun, 13 Mar 2022 17:16:22 +0100 Subject: [PATCH 1/6] re-arranged tests --- tests/{test_dataset.py => test_1001.py} | 46 ---------- tests/test_bulkIO.py | 117 ++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 46 deletions(-) rename tests/{test_dataset.py => test_1001.py} (86%) create mode 100644 tests/test_bulkIO.py diff --git a/tests/test_dataset.py b/tests/test_1001.py similarity index 86% rename from tests/test_dataset.py rename to tests/test_1001.py index e132a53..93b9259 100644 --- a/tests/test_dataset.py +++ b/tests/test_1001.py @@ -322,51 +322,5 @@ class Create1001TestCase(unittest.TestCase): return True - - -class BulkIOTestCase(unittest.TestCase): - # TODO: need to test invalid input - # def testInvalid(self): - # for fn in fns_fail: - # with self.subTest(msg=f"Opening invalid file {str(fn)}"): - # try: - # _ = icartt.Dataset(fn, loadData=True) - # except: # TODO: failure tests could be more specific - # pass - # else: - # self.fail('expected to fail') - - def testOpen(self): - for fn in fns_pass: - with self.subTest(msg=f"Opening test file {str(fn)}"): - ict = icartt.Dataset(fn, loadData=False) - self.assertEqual(type(ict), icartt.Dataset) - - def testReadData(self): - for fn in fns_pass: - with self.subTest(msg=f"Reading data from test file {str(fn)}"): - ict = icartt.Dataset(fn, loadData=True) - self.assertEqual(type(ict), icartt.Dataset) - - def testWriteHeader(self): - for fn in fns_pass: - with self.subTest(msg=f"Writing header for test file {str(fn)}"): - ict = icartt.Dataset(fn, loadData=False) - strIn = open(fn) - strOut = io.StringIO() - ict.writeHeader(f=strOut) - self.assertTrue(compareFiles(fn, strIn, strOut, nlines=ict.nHeader)) - - def testWrite(self): - for fn in fns_pass: - with self.subTest(msg=f"Writing data for test file {str(fn)}"): - ict = icartt.Dataset(fn, loadData=True) - strIn = open(fn) - strOut = io.StringIO() - ict.write(f=strOut) - self.assertTrue(compareFiles(fn, strIn, strOut)) - -# TODO: should also test dataset methods, e.g. .times - if __name__ == "__main__": # pragma: no cover unittest.main() diff --git a/tests/test_bulkIO.py b/tests/test_bulkIO.py new file mode 100644 index 0000000..29de779 --- /dev/null +++ b/tests/test_bulkIO.py @@ -0,0 +1,117 @@ +import unittest +import pathlib +import io +import icartt + +# working directory, example files +wd = pathlib.Path(__file__).parent +fns_pass = (wd / "example_data").glob("*.ict") +fns_fail = (wd / "example_data" / "will_fail").glob("*.ict") + + +def compareFiles(fn, strIn, strOut, skiplines=0, nlines=-1): # pragma: no cover + strOut.seek(0) + strIn.seek(0) + content_in = strIn.readlines() + content_out = strOut.readlines() + strIn.close() + strOut.close() + + if nlines > 0: + content_in = content_in[skiplines : (skiplines + nlines)] + content_out = content_out[skiplines : (skiplines + nlines)] + else: + content_in = content_in[skiplines:] + content_out = content_out[skiplines:] + + if not len(content_in) == len(content_out): + return False + + for inline, outline in zip(content_in, content_out): + inline = inline.strip().replace(" ", "") + outline = outline.strip().replace(" ", "") + if not inline == outline: + valid_data_line = False + # maybe this is a data line in which we only have different number formatting? + # compare as floats + # try: + insteps = [float(x) for x in inline.split(",")] + outsteps = [float(x) for x in outline.split(",")] + if len(insteps) == len(outsteps): + valid_data_line = True + for i in range(len(insteps)): + valid_data_line = valid_data_line and insteps[i] == outsteps[i] + # except: + # pass + + valid_var_line = False + # try: + insteps = [x.strip() for x in inline.split(",")] + outsteps = [x.strip() for x in outline.split(",")] + if len(insteps) == 2 and len(outsteps) == 3: + valid_var_line = ( + insteps[0] == outsteps[0] + and insteps[1] == outsteps[1] + and insteps[1] == outsteps[2] + ) + # except: + # pass + + if not valid_data_line and not valid_var_line: + print(f"{str(fn)}: line {i:d} differs:") + print(f" input: {inline}") + print(f" output: {outline}") + + return False + + return True + + + + +class BulkIOTestCase(unittest.TestCase): + # TODO: need to test invalid input + # def testInvalid(self): + # for fn in fns_fail: + # with self.subTest(msg=f"Opening invalid file {str(fn)}"): + # try: + # _ = icartt.Dataset(fn, loadData=True) + # except: # TODO: failure tests could be more specific + # pass + # else: + # self.fail('expected to fail') + + def testOpen(self): + for fn in fns_pass: + with self.subTest(msg=f"Opening test file {str(fn)}"): + ict = icartt.Dataset(fn, loadData=False) + self.assertEqual(type(ict), icartt.Dataset) + + def testReadData(self): + for fn in fns_pass: + with self.subTest(msg=f"Reading data from test file {str(fn)}"): + ict = icartt.Dataset(fn, loadData=True) + self.assertEqual(type(ict), icartt.Dataset) + + def testWriteHeader(self): + for fn in fns_pass: + with self.subTest(msg=f"Writing header for test file {str(fn)}"): + ict = icartt.Dataset(fn, loadData=False) + strIn = open(fn) + strOut = io.StringIO() + ict.writeHeader(f=strOut) + self.assertTrue(compareFiles(fn, strIn, strOut, nlines=ict.nHeader)) + + def testWrite(self): + for fn in fns_pass: + with self.subTest(msg=f"Writing data for test file {str(fn)}"): + ict = icartt.Dataset(fn, loadData=True) + strIn = open(fn) + strOut = io.StringIO() + ict.write(f=strOut) + self.assertTrue(compareFiles(fn, strIn, strOut)) + +# TODO: should also test dataset methods, e.g. .times + +if __name__ == "__main__": # pragma: no cover + unittest.main() -- GitLab From 7186960864d63288f38421522e451678a44eda83 Mon Sep 17 00:00:00 2001 From: Florian Obersteiner Date: Sun, 13 Mar 2022 17:17:35 +0100 Subject: [PATCH 2/6] formatting --- tests/test_1001.py | 4 ++-- tests/test_bulkIO.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_1001.py b/tests/test_1001.py index 93b9259..be343d1 100644 --- a/tests/test_1001.py +++ b/tests/test_1001.py @@ -311,9 +311,9 @@ class Create1001TestCase(unittest.TestCase): ict.data.addBulk(data) # elements of the time array must be equal to our input - t0 = np.datetime64(datetime.datetime(*now.timetuple()[:3]), 'ns') + t0 = np.datetime64(datetime.datetime(*now.timetuple()[:3]), "ns") for have, want in zip(ict.times, (12.3, 12.6, 13.4, 14.1)): - self.assertEqual(int(have-t0), int(want*10**9)) + self.assertEqual(int(have - t0), int(want * 10**9)) strOut = io.StringIO() diff --git a/tests/test_bulkIO.py b/tests/test_bulkIO.py index 29de779..d6e3697 100644 --- a/tests/test_bulkIO.py +++ b/tests/test_bulkIO.py @@ -67,8 +67,6 @@ def compareFiles(fn, strIn, strOut, skiplines=0, nlines=-1): # pragma: no cover return True - - class BulkIOTestCase(unittest.TestCase): # TODO: need to test invalid input # def testInvalid(self): @@ -111,6 +109,7 @@ class BulkIOTestCase(unittest.TestCase): ict.write(f=strOut) self.assertTrue(compareFiles(fn, strIn, strOut)) + # TODO: should also test dataset methods, e.g. .times if __name__ == "__main__": # pragma: no cover -- GitLab From c70b6fd58c63e4f185ec7eb08772e26789106a97 Mon Sep 17 00:00:00 2001 From: Florian Obersteiner Date: Sun, 13 Mar 2022 17:24:38 +0100 Subject: [PATCH 3/6] ignore UserWarnings in unit tests --- tests/test_1001.py | 7 ++++++- tests/test_bulkIO.py | 9 ++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/test_1001.py b/tests/test_1001.py index be343d1..8bd9af5 100644 --- a/tests/test_1001.py +++ b/tests/test_1001.py @@ -323,4 +323,9 @@ class Create1001TestCase(unittest.TestCase): if __name__ == "__main__": # pragma: no cover - unittest.main() + + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=UserWarning) + unittest.main() diff --git a/tests/test_bulkIO.py b/tests/test_bulkIO.py index d6e3697..887abbc 100644 --- a/tests/test_bulkIO.py +++ b/tests/test_bulkIO.py @@ -110,7 +110,10 @@ class BulkIOTestCase(unittest.TestCase): self.assertTrue(compareFiles(fn, strIn, strOut)) -# TODO: should also test dataset methods, e.g. .times - if __name__ == "__main__": # pragma: no cover - unittest.main() + + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=UserWarning) + unittest.main() -- GitLab From d4c78cb4dd800f4e21b5ddc6a2c6770b49933b20 Mon Sep 17 00:00:00 2001 From: Florian Obersteiner Date: Sun, 13 Mar 2022 17:44:41 +0100 Subject: [PATCH 4/6] normalComments nlines check --- tests/test_1001.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/test_1001.py b/tests/test_1001.py index 8bd9af5..7dcb971 100644 --- a/tests/test_1001.py +++ b/tests/test_1001.py @@ -295,6 +295,41 @@ class Create1001TestCase(unittest.TestCase): ict.specialComments.append("They are just examples!") ict.specialComments.append("Adapt as needed.") + # we can just use len of the list to check number of comments + self.assertEqual(len(ict.specialComments), 3) + + # let's define some normal comments... 21 lines + ncom = { + "PI_CONTACT_INFO": "PI1 pi-email@mail.com\nPI2 more-email@what.com", + "PLATFORM": "a platform", + "LOCATION": "somewhere", + "ASSOCIATED_DATA": "met sensor data", + "INSTRUMENT_INFO": "super cool instrument", + "DATA_INFO": f"icartt Python package version: {icartt.__version__}", + "UNCERTAINTY": "not much", + "ULOD_FLAG": "-7777", + "ULOD_VALUE": "N/A", + "LLOD_FLAG": "-8888", + "LLOD_VALUE": "N/A", + "DM_CONTACT_INFO": "datamanager@mail.edu", + "PROJECT_INFO": "the campaign", + "STIPULATIONS_ON_USE": "no", + "OTHER_COMMENTS": "a lot more info\non multiple lines", + "REVISION": ( + "R1\n" + "R1: revised time synchronization.\n" + "R0: initial, preliminary version." + ), + } + for k, v in ncom.items(): + ict.normalComments.keywords[k].append(v) + + # we can check if nlines method of normalComments class works + self.assertEqual(ict.normalComments.nlines, 21) + + ict.normalComments.freeform = ["free comment line"] + self.assertEqual(ict.normalComments.nlines, 22) + ict.endDefineMode() # we have not added data yet, so data must be None -- GitLab From 797e1da8711f0cccf5f46a9fe5811eaa092032fd Mon Sep 17 00:00:00 2001 From: Florian Obersteiner Date: Sun, 13 Mar 2022 17:55:52 +0100 Subject: [PATCH 5/6] examples adjusted to usage of tuple as dateOfCollection and dateOfRevision --- tests/usage_examples/create_ffi1001.py | 14 ++++++++++---- tests/usage_examples/create_ffi2110.py | 13 ++++++++----- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/tests/usage_examples/create_ffi1001.py b/tests/usage_examples/create_ffi1001.py index 21e4b8c..2261599 100644 --- a/tests/usage_examples/create_ffi1001.py +++ b/tests/usage_examples/create_ffi1001.py @@ -1,6 +1,7 @@ +import datetime import icartt -import datetime + ict = icartt.Dataset(format=icartt.Formats.FFI1001) @@ -8,8 +9,8 @@ ict.PIName = 'Knote, Christoph' ict.PIAffiliation = 'Faculty of Medicine, University Augsburg, Germany' ict.dataSourceDescription = 'Example data' ict.missionName = 'MBEES' -ict.dateOfCollection = datetime.datetime.today() -ict.dateOfRevision = datetime.datetime.today() +ict.dateOfCollection = datetime.datetime.utcnow().timetuple()[:3] +ict.dateOfRevision = datetime.datetime.utcnow().timetuple()[:3] ict.dataIntervalCode = [ 0 ] @@ -36,6 +37,11 @@ ict.specialComments.append("Some comments on this dataset:") ict.specialComments.append("They are just examples!") ict.specialComments.append("Adapt as needed.") +ict.normalComments.freeform.append('free comment line 1') +ict.normalComments.freeform.append('free comment line 2') + +# ict.normalComments are all set to N/A if not specified + ict.endDefineMode() # Three ways to add data: @@ -69,4 +75,4 @@ ict.write() # And you could simply write to file: #with open('output.ict', 'w') as f: -# ict.write(f=f) \ No newline at end of file +# ict.write(f=f) diff --git a/tests/usage_examples/create_ffi2110.py b/tests/usage_examples/create_ffi2110.py index 8796a22..78ca197 100644 --- a/tests/usage_examples/create_ffi2110.py +++ b/tests/usage_examples/create_ffi2110.py @@ -1,15 +1,18 @@ -import icartt import datetime + import numpy as np +import icartt + + ict = icartt.Dataset(format=icartt.Formats.FFI2110) ict.PIName = 'Knote, Christoph' ict.PIAffiliation = 'Faculty of Medicine, University Augsburg, Germany' ict.dataSourceDescription = 'Example data' ict.missionName = 'MBEES' -ict.dateOfCollection = datetime.datetime.today() -ict.dateOfRevision = datetime.datetime.today() +ict.dateOfCollection = datetime.datetime.utcnow().timetuple()[:3] +ict.dateOfRevision = datetime.datetime.utcnow().timetuple()[:3] ict.dataIntervalCode = [ 0 ] @@ -83,7 +86,7 @@ ict.data.add( Time_Start = 13.3, nAuxiliary=2, Time_Stop = 13.5, Latitude = 48.3 # then, dependent data can be added: # ibvar, dvar1, dvar2 -data = np.array( [ ( 0, 123, 8.4e4), +data = np.array( [ ( 0, 123, 8.4e4), (100, 122, 9.1e4), (250, 115, 9.3e4), (500, 106, 9.8e4) ] ) @@ -91,7 +94,7 @@ data = np.array( [ ( 0, 123, 8.4e4), ict.data.addBulkDep(12.3, data) # ibvar, dvar1, dvar2 -data = np.array( [ ( 0, 153, 7.3e4), +data = np.array( [ ( 0, 153, 7.3e4), (270, 172, 8.9e4) ] ) ict.data.addBulkDep(13.3, data) -- GitLab From 4a27b7681ee70aaf64dff058f2d5a9fd70ca8685 Mon Sep 17 00:00:00 2001 From: Florian Obersteiner Date: Sun, 13 Mar 2022 17:56:50 +0100 Subject: [PATCH 6/6] test adjusted --- tests/test_1001.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_1001.py b/tests/test_1001.py index 7dcb971..bbaf2f4 100644 --- a/tests/test_1001.py +++ b/tests/test_1001.py @@ -327,8 +327,9 @@ class Create1001TestCase(unittest.TestCase): # we can check if nlines method of normalComments class works self.assertEqual(ict.normalComments.nlines, 21) - ict.normalComments.freeform = ["free comment line"] - self.assertEqual(ict.normalComments.nlines, 22) + ict.normalComments.freeform.append("free comment line 1") + ict.normalComments.freeform.append("free comment line 2") + self.assertEqual(ict.normalComments.nlines, 23) ict.endDefineMode() -- GitLab