From bbd80004cf128e04ddaff71117e6bd9663f55d35 Mon Sep 17 00:00:00 2001
From: Christoph Knote <christoph.knote@med.uni-augsburg.de>
Date: Fri, 18 Feb 2022 16:23:26 +0100
Subject: [PATCH] Fix structure to match Python Packaging User Guide
---
CHANGES.md | 15 +
CHANGES.rst | 19 -
INSTALL.md | 1 +
INSTALL.rst | 3 -
MANIFEST.in | 8 +-
README.md | 12 +-
docs/source/conf.py | 2 +-
docs/source/index.rst | 19 +-
docs/source/usage.rst | 110 +----
pyproject.toml | 5 +
setup.cfg | 33 ++
setup.py | 35 --
{icartt => src/icartt}/__init__.py | 0
{icartt => src/icartt}/dataset.py | 410 +++++++++++-------
.../AR_DC8_20050203_R0.ict | 0
.../BB-FLUX_CU-SOF_20180808_R2.ict | 0
.../DC8-20160517.ict | 0
.../Dongdaemun_NIER_20160520_RA.ict | 0
.../HOX_DC8_20040712_R0.ict | 0
.../NOx_RHBrown_20040830_R0.ict | 0
.../PAVE-AR_DC8_20050203_R0.ict | 0
.../bt_Munich_2020061000_72.ict.txt | 0
...aq-flexpart-dc8_trajectory_20160529_R2.ict | 0
...rg01-HANSEO-KING-AIR_merge_20160507_RA.ict | 0
.../korusaq-mrg10-dc8_merge_20160510_R4.ict | 0
.../read_and_write.py | 0
.../will_fail/AROTAL-RAY_DC8_20040715_R1.ict | 0
.../will_fail/LIDARO3_WP3_20040830_R0.ict | 0
tests/test_dataset.py | 8 +-
tests/usage_examples/create_ffi1001.py | 72 +++
tests/usage_examples/create_ffi2110.py | 104 +++++
tests/usage_examples/read_ffi1001.py | 28 ++
tests/usage_examples/read_ffi2110.py | 34 ++
33 files changed, 595 insertions(+), 323 deletions(-)
create mode 100644 CHANGES.md
delete mode 100644 CHANGES.rst
create mode 100644 INSTALL.md
delete mode 100644 INSTALL.rst
create mode 100644 pyproject.toml
create mode 100644 setup.cfg
delete mode 100644 setup.py
rename {icartt => src/icartt}/__init__.py (100%)
rename {icartt => src/icartt}/dataset.py (76%)
rename tests/{examples => example_data}/AR_DC8_20050203_R0.ict (100%)
rename tests/{examples => example_data}/BB-FLUX_CU-SOF_20180808_R2.ict (100%)
rename tests/{examples => example_data}/DC8-20160517.ict (100%)
rename tests/{examples => example_data}/Dongdaemun_NIER_20160520_RA.ict (100%)
rename tests/{examples => example_data}/HOX_DC8_20040712_R0.ict (100%)
rename tests/{examples => example_data}/NOx_RHBrown_20040830_R0.ict (100%)
rename tests/{examples => example_data}/PAVE-AR_DC8_20050203_R0.ict (100%)
rename tests/{examples => example_data}/bt_Munich_2020061000_72.ict.txt (100%)
rename tests/{examples => example_data}/korusaq-flexpart-dc8_trajectory_20160529_R2.ict (100%)
rename tests/{examples => example_data}/korusaq-mrg01-HANSEO-KING-AIR_merge_20160507_RA.ict (100%)
rename tests/{examples => example_data}/korusaq-mrg10-dc8_merge_20160510_R4.ict (100%)
rename tests/{examples => example_data}/read_and_write.py (100%)
rename tests/{examples => example_data}/will_fail/AROTAL-RAY_DC8_20040715_R1.ict (100%)
rename tests/{examples => example_data}/will_fail/LIDARO3_WP3_20040830_R0.ict (100%)
create mode 100644 tests/usage_examples/create_ffi1001.py
create mode 100644 tests/usage_examples/create_ffi2110.py
create mode 100644 tests/usage_examples/read_ffi1001.py
create mode 100644 tests/usage_examples/read_ffi2110.py
diff --git a/CHANGES.md b/CHANGES.md
new file mode 100644
index 0000000..c754a6b
--- /dev/null
+++ b/CHANGES.md
@@ -0,0 +1,15 @@
+# Changelog
+
+## 2.0.0 (2022-02-x)
+
+- Compatible with ICARTT v2 standard
+- Formats 1001 and 2110
+- Complete internal overhaul
+
+## 1.0.0 (2017-12-19)
+
+- Peer-reviewed version to be published in Knote et al., GMD
+
+## 0.1.0 (2017-08-12)
+
+- Initial release
diff --git a/CHANGES.rst b/CHANGES.rst
deleted file mode 100644
index 9461260..0000000
--- a/CHANGES.rst
+++ /dev/null
@@ -1,19 +0,0 @@
-Changelog
-=========
-
-2.0.0 (2022-02-x)
-------------------
-
-- Compatible with ICARTT v2 standard
-- Formats 1001 and 2110
-- Internal overhaul
-
-1.0.0 (2017-12-19)
-------------------
-
-- Peer-reviewed version to be published in Knote et al., GMD
-
-0.1.0 (2017-08-12)
-------------------
-
-- Initial release
diff --git a/INSTALL.md b/INSTALL.md
new file mode 100644
index 0000000..6c39ecc
--- /dev/null
+++ b/INSTALL.md
@@ -0,0 +1 @@
+`pip install icartt`
\ No newline at end of file
diff --git a/INSTALL.rst b/INSTALL.rst
deleted file mode 100644
index 1835242..0000000
--- a/INSTALL.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-::
-
- pip install icartt
diff --git a/MANIFEST.in b/MANIFEST.in
index 1d3bef9..06b0b69 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,4 +1,4 @@
-include CHANGES.rst
-include INSTALL.rst
-include LICENSE.txt
-include README.rst
+include CHANGES.md
+include INSTALL.md
+include LICENSE
+include README.md
diff --git a/README.md b/README.md
index 5f972b4..ed717dd 100644
--- a/README.md
+++ b/README.md
@@ -1,15 +1,12 @@
-icartt
-======
+# icartt
``icartt`` is an ICARTT format reader and writer
-Documentation
-=============
+## Documentation
Please have a look at docs/source/usage.rst for usage examples. Full documentation is in preparation.
-Contributing
-============
+## Contributing
We are happy to receive your [new issue report](https://mbees.med.uni-augsburg.de/gitlab/mbees/icartt_pypackage/-/issues/new).
@@ -17,8 +14,7 @@ If you'd like to contribute source code directly, please [create a fork](https:/
make your changes and then [submit a merge request](https://mbees.med.uni-augsburg.de/gitlab/mbees/icartt_pypackage/-/merge_requests/new) to the original project.
-Installation
-============
+## Installation
Clone this repository / or your fork and install as "editable":
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 3a9e503..4519d63 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -46,7 +46,7 @@ exclude_patterns = []
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
-html_theme = 'alabaster'
+html_theme = 'classic'
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
diff --git a/docs/source/index.rst b/docs/source/index.rst
index a256cfe..27fead2 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -3,8 +3,9 @@
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
+******
icartt
-============================
+******
icartt is an ICARTT file format reader and writer for Python
@@ -15,39 +16,39 @@ The ICARTT data format is described here: https://www-air.larc.nasa.gov/missions
:caption: Contents:
Installation
-------------
+############
-.. include:: ../../INSTALL.rst
+.. include:: ../../INSTALL.md
Example
-------------
+#######
.. include:: usage.rst
API
-----
+###
.. module:: icartt
Variable
-^^^^^^^^^^
+********
.. autoclass:: Variable
:members:
Dataset
-^^^^^^^^^^
+********
.. autoclass:: Dataset
:members:
Formats
-^^^^^^^^^^
+********
.. autoenum:: Formats
Indices and tables
-==================
+###################
* :ref:`genindex`
* :ref:`modindex`
diff --git a/docs/source/usage.rst b/docs/source/usage.rst
index 8e81302..c5eb0ac 100644
--- a/docs/source/usage.rst
+++ b/docs/source/usage.rst
@@ -1,103 +1,29 @@
-Reading an existing dataset
-^^^^^^^^^^^^^^^^^^^^^^^^^^
+Reading an existing dataset
+############################
-::
+Simple format (FFI 1001)
+*************************
- import icartt
+.. literalinclude:: ../../tests/usage_examples/read_ffi1001.py
- # load a new dataset from an existing file
- ict = icartt.Dataset('tests/examples/DC8-20160517.ict')
+More complex (FFI 2110)
+*************************
- # list variable names
- [ x for x in ict.variables ]
+Identical to FFI1001, only the data structure is more complex:
- # get data for variable 'UTC' (shortcut):
- ict.data['UTC']
+.. literalinclude:: ../../tests/usage_examples/read_ffi2110.py
- # get all data as NumPy array:
- ict.data.data
+Creating a new dataset
+############################
- # read some metadata
- ict.PIName
- ict.PIAffiliation
- ict.missionName
- ict.dataSourceDescription
+Simple format (FFI 1001)
+*************************
- # some info on a variable
- ict.variables['Alt_ft'].units
- ict.variables['Alt_ft'].miss
+.. literalinclude:: ../../tests/usage_examples/create_ffi1001.py
+More complex (FFI 2110)
+*************************
-Creating a new dataset
-^^^^^^^^^^^^^^^^^^^^^^^
+Again, like for FFI 1001 but more complex data structure
-::
-
- import icartt
- import datetime
-
- ict = icartt.Dataset(format=icartt.Formats.FFI1001)
-
- ict.PIName = 'Knote, Christoph'
- ict.PIAffiliation = 'Faculty of Medicine, University Augsburg, Germany'
- ict.dataSourceDescription = 'Example data'
- ict.missionName = 'MBEES'
- ict.dateOfCollection = datetime.datetime.today()
- ict.dateOfRevision = datetime.datetime.today()
-
- ict.dataIntervalCode = [ 0 ]
-
- ict.independentVariable = icartt.Variable( 'Time_Start',
- 'seconds_from_0_hours_on_valid_date',
- 'Time_Start',
- 'Time_Start',
- vartype=icartt.VariableType.IndependentVariable,
- scale=1.0, miss=-9999999)
-
- ict.dependentVariables['Time_Stop'] = icartt.Variable( 'Time_Stop',
- 'seconds_from_0_hours_on_valid_date',
- 'Time_Stop',
- 'Time_Stop',
- scale=1.0, miss=-9999999)
-
- ict.dependentVariables['Payload'] = icartt.Variable( 'Payload',
- 'some_units',
- 'Payload',
- 'Payload',
- scale=1.0, miss=-9999999)
-
- ict.specialComments.append("Some comments on this dataset:")
- ict.specialComments.append("They are just examples!")
- ict.specialComments.append("Adapt as needed.")
-
- ict.endDefineMode()
-
- # Three ways to add data:
-
- # 1) simple (single data line)
- ict.data.add( Time_Start = 12.3, Time_Stop = 12.5, Payload = 23789423.2e5 )
-
- # Let's check:
- ict.write()
-
- # 2) as dictionary (single data line)
- mydict = { 'Time_Start': 12.6, 'Time_Stop': 13.1, 'Payload': 324235644.1e5 }
- ict.data.add( **mydict )
- # (note, exploding the dictionary is necessary)
-
- # 3) as NumPy array (bulk)
- import numpy as np
- data = np.array( [ (13.4, 14.0, 2348925e5), (14.1, 14.9, 23425634e5) ] )
- ict.data.addBulk( data )
-
- # Note 1: you are responsible to ensure that the order of elements in a data line
- # corresponds to variable listing below:
- print( [ x for x in ict.variables ] )
-
- # Note 2: for single lines, you still need to make it an array!
- data = np.array( [ (15.4, 15.0, 52452495290e5) ] )
- ict.data.addBulk( data )
-
- # Now write to file:
- with open('path/to/output.ict', 'w') as f:
- ict.write(f=f)
+.. literalinclude:: ../../tests/usage_examples/create_ffi1001.py
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..d9c84a6
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,5 @@
+[build-system]
+requires = [
+ "numpy", "setuptools"
+]
+build-backend = "setuptools.build_meta"
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 0000000..78485d1
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,33 @@
+[metadata]
+name = icartt
+version = 1.9.1
+author = Christoph Knote
+author_email = christoph.knote@med.uni-augsburg.de
+description = ICARTT format reader and writer
+long_description = file: README.md, INSTALL.md, CHANGES.md
+long_description_content_type = text/markdown
+url = https://mbees.med.uni-augsburg.de
+project_urls =
+ Bug Tracker = http://mbees.med.uni-augsburg.de/gitlab/mbees/icartt_pypackage/issues
+classifiers =
+ Programming Language :: Python :: 3
+ Development Status :: 5 - Production/Stable
+ Environment :: Console
+ Intended Audience :: Developers
+ Intended Audience :: Education
+ Intended Audience :: End Users/Desktop
+ Intended Audience :: Science/Research
+ License :: OSI Approved :: GNU General Public License v3 (GPLv3)
+ Operating System :: POSIX
+ Topic :: Education
+ Topic :: Scientific/Engineering
+ Topic :: Utilities
+
+[options]
+package_dir =
+ = src
+packages = find:
+python_requires = >=3.0
+
+[options.packages.find]
+where = src
diff --git a/setup.py b/setup.py
deleted file mode 100644
index 6130b48..0000000
--- a/setup.py
+++ /dev/null
@@ -1,35 +0,0 @@
-import os
-from setuptools import setup
-
-def read(filename):
- with open(os.path.join(os.path.dirname(__file__), filename)) as f:
- return f.read()
-
-setup(name='icartt',
- description='ICARTT format reader and writer',
- long_description=read('README.md') + '\n\n' + read('INSTALL.rst') + '\n\n' + read('CHANGES.rst'),
- long_description_content_type='text/markdown',
- version='1.9.1',
- url='https://mbees.med.uni-augsburg.de',
- author='Christoph Knote',
- author_email='christoph.knote@med.uni-augsburg.de',
- license='GPLv3',
- classifiers=[
- 'Development Status :: 5 - Production/Stable',
- 'Environment :: Console',
- 'Intended Audience :: Developers',
- 'Intended Audience :: Education',
- 'Intended Audience :: End Users/Desktop',
- 'Intended Audience :: Science/Research',
- 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
- 'Operating System :: POSIX',
- 'Programming Language :: Python :: 3',
- 'Topic :: Education',
- 'Topic :: Scientific/Engineering',
- 'Topic :: Utilities'
- ],
- keywords='',
- packages=['icartt'],
- test_suite = 'tests',
- tests_require = [],
- zip_safe=False)
diff --git a/icartt/__init__.py b/src/icartt/__init__.py
similarity index 100%
rename from icartt/__init__.py
rename to src/icartt/__init__.py
diff --git a/icartt/dataset.py b/src/icartt/dataset.py
similarity index 76%
rename from icartt/dataset.py
rename to src/icartt/dataset.py
index c46c5b2..88d63fb 100644
--- a/icartt/dataset.py
+++ b/src/icartt/dataset.py
@@ -10,12 +10,15 @@ from enum import Enum, IntEnum
DEFAULT_NUM_FORMAT = "%f"
DEFAULT_FIELD_DELIM = ", "
+
+
class Formats(IntEnum):
- '''File Format Indicators (FFI)
- '''
+ """File Format Indicators (FFI)"""
+
FFI1001 = 1001
FFI2110 = 2110
+
class VariableType(Enum):
IndependentVariable = 1
IndependentBoundedVariable = 2
@@ -26,27 +29,25 @@ class VariableType(Enum):
def sanitize(val, miss):
return float(val) if not float(val) == float(miss) else np.NaN
+
class DataStore1001:
def __init__(self, ivar, dvars):
self.ivarname = ivar.shortname
-#
+
self.varnames = [ivar.shortname] + [x for x in dvars]
self.missvals = {x: dvars[x].miss for x in dvars}
self.missvals.update({self.ivarname: ivar.miss})
-#
+
self.data = None
-#
def __getitem__(self, s=slice(None)):
return self.data[s]
-#
def addBulk(self, raw):
nlines, nvars = raw.shape
if not nvars == len(self.varnames):
raise Exception("Number of data columns does not match variable count!")
self._addBulk(raw, nlines)
-#
def addBulkFromTxt(self, raw):
if not len(raw[0]) == len(self.varnames):
@@ -57,15 +58,14 @@ class DataStore1001:
for cur in range(n):
newdata = {x: raw[cur][i] for i, x in enumerate(self.varnames)}
self.add(**newdata)
-#
+
def add(self, **kwargs):
if not self.ivarname in kwargs.keys():
raise Exception("Need independent variable data.")
- ivarvalue = sanitize(kwargs[self.ivarname],
- self.missvals[self.ivarname])
+ ivarvalue = sanitize(kwargs[self.ivarname], self.missvals[self.ivarname])
- newline = np.array(np.NaN, dtype=[(v, 'f8') for v in self.varnames])
+ newline = np.array(np.NaN, dtype=[(v, "f8") for v in self.varnames])
for key in kwargs.keys():
if key in self.varnames:
newline[key] = sanitize(kwargs[key], self.missvals[key])
@@ -85,11 +85,14 @@ class DataStore1001:
dd[k][np.isnan(dd[k])] = miss
return dd
- def write(self, f=sys.stdout, fmt=DEFAULT_NUM_FORMAT, delimiter=DEFAULT_FIELD_DELIM):
+ def write(
+ self, f=sys.stdout, fmt=DEFAULT_NUM_FORMAT, delimiter=DEFAULT_FIELD_DELIM
+ ):
d = self.denanify(self.data)
np.savetxt(f, d, fmt=fmt, delimiter=delimiter)
-class DataStore2110:
+
+class DataStore2110(collections.UserDict):
def __init__(self, ivar, ibvar, auxvars, dvars):
self.ivarname = ivar.shortname
self.ibvarname = ibvar.shortname
@@ -103,51 +106,62 @@ class DataStore2110:
self.missvals.update({self.ivarname: ivar.miss})
self.nauxvarname = self.auxvarnames[0] # convention!
-#
+
self.data = {}
-#
+
self.ivar = ivar
self.auxvars = auxvars
self.ibvar = ibvar
self.dvars = dvars
-#
+
+ def __getitem__(self, s=slice(None)):
+ return self.data[s]
- def addAuxline(self, auxline):
- newdata = {x: auxline[i] for i, x in enumerate(
- [self.ivarname] + self.auxvarnames)}
+ def _addAuxline(self, auxline):
+ newdata = {
+ x: auxline[i] for i, x in enumerate([self.ivarname] + self.auxvarnames)
+ }
self.add(**newdata)
- def addDeplines(self, ivar, raw):
- for cur in range(len(raw)):
- newdata = {x: raw[cur][i] for i, x in enumerate(
- [self.ibvarname] + self.dvarnames)}
+ def addBulkDep(self, ivar, raw):
+ nlines, nvars = raw.shape
+ self._addDeplines(ivar, raw, nlines)
+
+ def _addDeplines(self, ivar, raw, n):
+ for cur in range(n):
+ newdata = {
+ x: raw[cur][i] for i, x in enumerate([self.ibvarname] + self.dvarnames)
+ }
newdata.update({self.ivarname: ivar})
self.add(**newdata)
def addBulkFromTxt(self, raw):
+ self._addBulk(raw, len(raw))
+
+ def _addBulk(self, raw, n):
cur = 0
- while cur < len(raw):
+ while cur < n:
ivarvalue = sanitize(raw[cur][0], self.missvals[self.ivarname])
- self.addAuxline(raw[cur])
+ self._addAuxline(raw[cur])
cur += 1
# stupid, but at first auxline added, nprimaryData ist a 0-dim array...
- nprimaryData = self.data[ivarvalue]['AUX'][self.nauxvarname]
- nprimary = int(nprimaryData) if nprimaryData.shape == () else int(
- nprimaryData[-1])
+ ndepData = self.data[ivarvalue]["AUX"][self.nauxvarname]
+ ndepData = (
+ int(ndepData) if ndepData.shape == () else int(ndepData[-1])
+ )
- self.addDeplines(ivarvalue, raw[cur:(cur+nprimary)])
+ self._addDeplines(ivarvalue, raw[cur : (cur + ndepData)], ndepData)
- cur += nprimary
+ cur += ndepData
def add(self, **kwargs):
# whatever we do, an independent variable is needed
if not self.ivarname in kwargs.keys():
raise Exception("Need independent variable data.")
- ivarvalue = sanitize(kwargs[self.ivarname],
- self.missvals[self.ivarname])
+ ivarvalue = sanitize(kwargs[self.ivarname], self.missvals[self.ivarname])
# this is an AUX line
if any([x in self.auxvarnames for x in kwargs.keys()]):
@@ -155,9 +169,9 @@ class DataStore2110:
if not ivarvalue in self.data.keys():
self.data[ivarvalue] = {
"AUX": DataStore1001(self.ivar, self.auxvars),
- "DEP": DataStore1001(self.ibvar, self.dvars)
+ "DEP": DataStore1001(self.ibvar, self.dvars),
}
- self.data[ivarvalue]['AUX'].add(**kwargs)
+ self.data[ivarvalue]["AUX"].add(**kwargs)
# this is a DEP line
if any([x in self.dvarnames for x in kwargs.keys()]):
@@ -167,14 +181,17 @@ class DataStore2110:
if not ivarvalue in self.data.keys():
raise Exception("Aux data line needs to be added first.")
- self.data[ivarvalue]['DEP'].add(**kwargs)
+ self.data[ivarvalue]["DEP"].add(**kwargs)
- def write(self, f=sys.stdout, fmt=DEFAULT_NUM_FORMAT, delimiter=DEFAULT_FIELD_DELIM):
+ def write(
+ self, f=sys.stdout, fmt=DEFAULT_NUM_FORMAT, delimiter=DEFAULT_FIELD_DELIM
+ ):
for ivarvalue in self.data:
- self.data[ivarvalue]['AUX'].write(f, fmt=fmt, delimiter=delimiter)
- self.data[ivarvalue]['DEP'].write(f, fmt=fmt, delimiter=delimiter)
+ self.data[ivarvalue]["AUX"].write(f, fmt=fmt, delimiter=delimiter)
+ self.data[ivarvalue]["DEP"].write(f, fmt=fmt, delimiter=delimiter)
-class KeywordComment():
+
+class KeywordComment:
def __init__(self, key, naAllowed):
self.key = key
self.naAllowed = naAllowed
@@ -187,15 +204,20 @@ class KeywordComment():
d = "\n".join(self.data) if not self.data is [] else "N/A"
return self.key + ": " + d
+
class StandardNormalComments(collections.UserList):
@property
def nlines(self):
# "+ 1" -> shortnames line, and keywords might be multiline...
- return len(self.freeform) + 1 + sum([len(k.data) for k in self.keywords.values()])
+ return (
+ len(self.freeform) + 1 + sum([len(k.data) for k in self.keywords.values()])
+ )
@property
def data(self):
- return self.freeform + [str(s) for s in self.keywords.values()] + [self.shortnames]
+ return (
+ self.freeform + [str(s) for s in self.keywords.values()] + [self.shortnames]
+ )
def ingest(self, raw):
# last line is always shortname
@@ -211,22 +233,27 @@ class StandardNormalComments(collections.UserList):
currentKeyword = None
for l in raw:
possibleKeyword = l.split(":")[0].strip()
- if possibleKeyword in self.keywords or re.match("R[a-zA-Z0-9]{1,2}[ ]*", possibleKeyword):
+ if possibleKeyword in self.keywords or re.match(
+ "R[a-zA-Z0-9]{1,2}[ ]*", possibleKeyword
+ ):
currentKeyword = possibleKeyword
if not currentKeyword in self.keywords: # for the revisions only...
self.keywords[currentKeyword] = KeywordComment(
- currentKeyword, False)
+ currentKeyword, False
+ )
if currentKeyword is None:
self.freeform.append(l)
else:
self.keywords[currentKeyword].append(
- l.replace(l.split(":")[0] + ":", "").strip())
+ l.replace(l.split(":")[0] + ":", "").strip()
+ )
for key in self.keywords:
if self.keywords[key].data == []:
warnings.warn(
- "Normal comments: required keyword {:s} is missing.".format(key))
+ "Normal comments: required keyword {:s} is missing.".format(key)
+ )
def __init__(self):
self.freeform = []
@@ -248,7 +275,7 @@ class StandardNormalComments(collections.UserList):
"PROJECT_INFO",
"STIPULATIONS_ON_USE",
"OTHER_COMMENTS",
- "REVISION"
+ "REVISION",
)
self.keywords = {k: KeywordComment(k, True) for k in requiredKeywords}
@@ -256,8 +283,9 @@ class StandardNormalComments(collections.UserList):
self.keywords["UNCERTAINTY"].naAllowed = False
self.keywords["REVISION"].naAllowed = False
+
class Variable:
- '''An ICARTT variable description with name, units, scale and missing value.
+ """An ICARTT variable description with name, units, scale and missing value.
:param shortname: Short name of the variable
:type shortname: str
@@ -279,14 +307,14 @@ class Variable:
:param miss: Missing value for the variable
:type miss: float, defaults to -99999.0
- '''
+ """
def desc(self, splitChar=","):
- '''Variable description string as it appears in an ICARTT file
+ """Variable description string as it appears in an ICARTT file
:return: description string
:rtype: str
- '''
+ """
descstr = [str(self.shortname), str(self.units)]
if not self.standardname is None:
descstr += [str(self.standardname)]
@@ -301,21 +329,32 @@ class Variable:
# and underscores.
def isAsciiAlphaOrUnderscore(x):
return re.match("[a-zA-Z0-9_]", x)
- allAreAlphaOrUnderscore = all(
- [isAsciiAlphaOrUnderscore(x) for x in name])
+
+ allAreAlphaOrUnderscore = all([isAsciiAlphaOrUnderscore(x) for x in name])
# The first character must be a letter,
firstIsAlpha = bool(re.match("[a-zA-Z]", name[0]))
# and the name can be at most 31 characters in length.
lessThan31Chars = len(name) <= 31
- return (allAreAlphaOrUnderscore and firstIsAlpha and lessThan31Chars)
-
- def __init__(self, shortname, units, standardname, longname, vartype=VariableType.DependentVariable, scale=1.0, miss=-99999.0):
- '''Constructor method
- '''
+ return allAreAlphaOrUnderscore and firstIsAlpha and lessThan31Chars
+
+ def __init__(
+ self,
+ shortname,
+ units,
+ standardname,
+ longname,
+ vartype=VariableType.DependentVariable,
+ scale=1.0,
+ miss=-99999.0,
+ ):
+ """Constructor method"""
if not self.isValidVariablename(shortname):
warnings.warn(
- "Variable short name {:s} does not comply with ICARTT standard v2".format(shortname))
+ "Variable short name {:s} does not comply with ICARTT standard v2".format(
+ shortname
+ )
+ )
self.shortname = shortname
self.units = units
@@ -328,8 +367,9 @@ class Variable:
def __repr__(self):
return "ICARTT Variable description"
+
class Dataset:
- '''An ICARTT dataset that can be created from scratch or read from a file,
+ """An ICARTT dataset that can be created from scratch or read from a file,
manipulated, and then written to a file.
:param f: file path or file handle to use
@@ -342,53 +382,69 @@ class Dataset:
:type splitChar: str, defaults to ","
:param format:
- '''
+ """
+
@property
def nHeader(self):
- '''Header line count
+ """Header line count
:return: line count
:rtype: int
- '''
+ """
total = -1
if self.format == Formats.FFI1001:
- total = 14 + len(self.dependentVariables) + \
- len(self.specialComments) + self.normalComments.nlines
+ total = (
+ 14
+ + len(self.dependentVariables)
+ + len(self.specialComments)
+ + self.normalComments.nlines
+ )
if self.format == Formats.FFI2110:
# 2: IVAR + IBVAR
- total = 16 + 2 + len(self.auxiliaryVariables) + len(self.dependentVariables) +\
- len(self.specialComments) + self.normalComments.nlines
+ total = (
+ 16
+ + 2
+ + len(self.auxiliaryVariables)
+ + len(self.dependentVariables)
+ + len(self.specialComments)
+ + self.normalComments.nlines
+ )
return total
@property
def times(self):
- '''Time steps of the data
+ """Time steps of the data
:return: list of time steps
:rtype: list
- '''
- return [self.dateOfCollection + datetime.timedelta(seconds=x) for x in self.independentVariable]
+ """
+ return [
+ self.dateOfCollection + datetime.timedelta(seconds=x)
+ for x in self.independentVariable
+ ]
@property
def variables(self):
- '''Variables (independent + dependent + auxiliary)
+ """Variables (independent + dependent + auxiliary)
:return: dictionary of all variables
:rtype: dict of Variable(s)
- '''
+ """
vars = {}
if not self.independentVariable is None:
vars[self.independentVariable.shortname] = self.independentVariable
if not self.independentBoundedVariable is None:
- vars[self.independentBoundedVariable.shortname] = self.independentBoundedVariable
+ vars[
+ self.independentBoundedVariable.shortname
+ ] = self.independentBoundedVariable
vars = {**vars, **self.dependentVariables, **self.auxiliaryVariables}
return vars
def readHeader(self, splitChar=","):
- '''Read the ICARTT header (from file)
- '''
+ """Read the ICARTT header (from file)"""
+
class FilehandleWithLinecounter:
def __init__(self, f, splitChar):
self.f = f
@@ -397,10 +453,9 @@ class Dataset:
def readline(self, doSplit=True):
self.line += 1
- dmp = self.f.readline().replace('\n', '').replace('\r', '')
+ dmp = self.f.readline().replace("\n", "").replace("\r", "")
if doSplit:
- dmp = [word.strip(' ')
- for word in dmp.split(self.splitChar)]
+ dmp = [word.strip(" ") for word in dmp.split(self.splitChar)]
return dmp
if self.inputFhandle.closed:
@@ -426,8 +481,7 @@ class Dataset:
try:
self.format = Formats(int(dmp[1]))
except:
- raise ValueError(
- "ICARTT format {:d} not implemented".format(dmp[1]))
+ raise ValueError("ICARTT format {:d} not implemented".format(dmp[1]))
if len(dmp) > 2:
self.version = dmp[2]
@@ -456,9 +510,11 @@ class Dataset:
# - comma delimited (yyyy, mm, dd, yyyy, mm, dd).
dmp = f.readline()
self.dateOfCollection = datetime.datetime.strptime(
- "".join(["{:s}".format(x) for x in dmp[0:3]]), '%Y%m%d')
+ "".join(["{:s}".format(x) for x in dmp[0:3]]), "%Y%m%d"
+ )
self.dateOfRevision = datetime.datetime.strptime(
- "".join(["{:s}".format(x) for x in dmp[3:6]]), '%Y%m%d')
+ "".join(["{:s}".format(x) for x in dmp[3:6]]), "%Y%m%d"
+ )
# line 8 - Data Interval (This value describes the time spacing (in seconds)
# between consecutive data records. It is the (constant) interval between
@@ -488,13 +544,23 @@ class Dataset:
if self.format == Formats.FFI2110:
dmp = f.readline()
shortname, units, standardname, longname = extractVardesc(dmp)
- self.independentBoundedVariable = Variable(shortname, units, standardname, longname,
- vartype=VariableType.IndependentBoundedVariable)
+ self.independentBoundedVariable = Variable(
+ shortname,
+ units,
+ standardname,
+ longname,
+ vartype=VariableType.IndependentBoundedVariable,
+ )
dmp = f.readline()
shortname, units, standardname, longname = extractVardesc(dmp)
- self.independentVariable = Variable(shortname, units, standardname, longname,
- vartype=VariableType.IndependentVariable)
+ self.independentVariable = Variable(
+ shortname,
+ units,
+ standardname,
+ longname,
+ vartype=VariableType.IndependentVariable,
+ )
def readVars(f, vtype):
# line 10 - Number of variables (Integer value showing the number of
@@ -504,12 +570,12 @@ class Dataset:
# line 11- Scale factors (1 for most cases, except where grossly
# inconvenient) - comma delimited.
- vscale = [ x for x in f.readline()]
+ vscale = [x for x in f.readline()]
# line 12 - Missing data indicators (This is -9999 (or -99999, etc.) for
# any missing data condition, except for the main time (independent)
# variable which is never missing) - comma delimited.
- vmiss = [ x for x in f.readline()]
+ vmiss = [x for x in f.readline()]
# no float casting here, as we need to do string comparison lateron when reading data...
# line 13 - Variable names and units (Short variable name and units are
@@ -534,13 +600,25 @@ class Dataset:
vstandardname += [standardname]
vlongname += [longname]
- return {shortname: Variable(shortname, unit, standardname, longname, scale=scale, miss=miss, vartype=vtype) for shortname, unit, standardname, longname, scale, miss in zip(vshortname, vunits, vstandardname, vlongname, vscale, vmiss)}
+ return {
+ shortname: Variable(
+ shortname,
+ unit,
+ standardname,
+ longname,
+ scale=scale,
+ miss=miss,
+ vartype=vtype,
+ )
+ for shortname, unit, standardname, longname, scale, miss in zip(
+ vshortname, vunits, vstandardname, vlongname, vscale, vmiss
+ )
+ }
self.dependentVariables = readVars(f, VariableType.DependentVariable)
if self.format == Formats.FFI2110:
- self.auxiliaryVariables = readVars(
- f, VariableType.AuxiliaryVariable)
+ self.auxiliaryVariables = readVars(f, VariableType.AuxiliaryVariable)
# line 14 + nvar - Number of SPECIAL comment lines (Integer value
# indicating the number of lines of special comments, NOT including this
@@ -550,8 +628,7 @@ class Dataset:
# line 15 + nvar - Special comments (Notes of problems or special
# circumstances unique to this file. An example would be comments/problems
# associated with a particular flight.).
- self.specialComments = [f.readline(
- doSplit=False) for i in range(0, nscom)]
+ self.specialComments = [f.readline(doSplit=False) for i in range(0, nscom)]
# line 16 + nvar + nscom - Number of Normal comments (i.e., number of
# additional lines of SUPPORTING information: Integer value indicating the
@@ -579,18 +656,19 @@ class Dataset:
self.nHeaderFile = f.line
if self.nHeader != nHeaderSuggested:
- warnings.warn("Number of header lines suggested in line 1 ({:d}) do not match actual header lines read ({:d})".format(
- nHeaderSuggested, self.nHeader))
+ warnings.warn(
+ "Number of header lines suggested in line 1 ({:d}) do not match actual header lines read ({:d})".format(
+ nHeaderSuggested, self.nHeader
+ )
+ )
def readData(self, splitChar=","):
- '''Read ICARTT data (from file)
- '''
+ """Read ICARTT data (from file)"""
if self.inputFhandle.closed:
self.inputFhandle = open(self.inputFhandle.name)
try:
- nul = [self.inputFhandle.readline()
- for i in range(self.nHeaderFile)]
+ nul = [self.inputFhandle.readline() for i in range(self.nHeaderFile)]
raw = [line.split(splitChar) for line in self.inputFhandle]
@@ -601,27 +679,34 @@ class Dataset:
self.inputFhandle.close()
def read(self, splitChar=","):
- '''Read ICARTT data and header
- '''
+ """Read ICARTT data and header"""
self.readHeader(splitChar)
self.endDefineMode(splitChar)
self.readData(splitChar)
- def makeFileName(self, dateFormat='%Y%m%d'):
- '''Create ICARTT-compliant file name based on the information contained in the dataset
+ def makeFileName(self, dateFormat="%Y%m%d"):
+ """Create ICARTT-compliant file name based on the information contained in the dataset
:param dateFormat: date format to use when parsing
:type dateFormat: str, defaults to '%Y%m%d'
:return: file name generated
:rtype: string
- '''
- fn = self.dataID + "_" + self.locationID + "_" + \
- datetime.datetime.strftime(self.dateOfCollection, dateFormat)
+ """
+ fn = (
+ self.dataID
+ + "_"
+ + self.locationID
+ + "_"
+ + datetime.datetime.strftime(self.dateOfCollection, dateFormat)
+ )
fn += "_R" + str(self.revision) if not self.revision is None else ""
fn += "_L" + str(self.launch) if not self.launch is None else ""
- fn += "_V" + \
- str(self.fileVolumeNumber) if self.totalNumberOfFileVolumes > 1 else ""
+ fn += (
+ "_V" + str(self.fileVolumeNumber)
+ if self.totalNumberOfFileVolumes > 1
+ else ""
+ )
return fn + ".ict"
@@ -633,17 +718,19 @@ class Dataset:
# characters in length.
def isAsciiAlpha(x):
return re.match("[a-zA-Z0-9-_.]", x)
+
allAsciiAlpha = all([isAsciiAlpha(x) for x in name])
lessThan128Characters = len(name) < 128
return allAsciiAlpha and lessThan128Characters
def writeHeader(self, f=sys.stdout, delimiter=DEFAULT_FIELD_DELIM):
- '''Write header
+ """Write header
:param f: handle to write to
:type f: file handle or StringIO stream, defaults to sys.stdout
- '''
+ """
+
def prnt(txt):
f.write(str(txt) + "\n")
@@ -663,11 +750,20 @@ class Dataset:
# Mission name (usually the mission acronym).
prnt(self.missionName)
# File volume number, number of file volumes (these integer values are used when the data require more than one file per day; for data that require only one file these values are set to 1, 1) - comma delimited.
- prnt(delimiter.join([str(self.fileVolumeNumber),
- str(self.totalNumberOfFileVolumes)]))
+ prnt(
+ delimiter.join(
+ [str(self.fileVolumeNumber), str(self.totalNumberOfFileVolumes)]
+ )
+ )
# UTC date when data begin, UTC date of data reduction or revision - comma delimited (yyyy, mm, dd, yyyy, mm, dd).
- prnt(delimiter.join([datetime.datetime.strftime(x, delimiter.join(
- ["%Y", "%m", "%d"])) for x in [self.dateOfCollection, self.dateOfRevision]]))
+ prnt(
+ delimiter.join(
+ [
+ datetime.datetime.strftime(x, delimiter.join(["%Y", "%m", "%d"]))
+ for x in [self.dateOfCollection, self.dateOfRevision]
+ ]
+ )
+ )
# Data Interval (This value describes the time spacing (in seconds) between consecutive data records. It is the (constant) interval between values of the independent variable. For 1 Hz data the data interval value is 1 and for 10 Hz data the value is 0.1. All intervals longer than 1 second must be reported as Start and Stop times, and the Data Interval value is set to 0. The Mid-point time is required when it is not at the average of Start and Stop times. For additional information see Section 2.5 below.).
prnt(delimiter.join([str(x) for x in self.dataIntervalCode]))
if self.format == Formats.FFI2110:
@@ -678,24 +774,39 @@ class Dataset:
# Number of variables (Integer value showing the number of dependent variables: the total number of columns of data is this value plus one.).
prnt(len(self.dependentVariables))
# Scale factors (1 for most cases, except where grossly inconvenient) - comma delimited.
- prnt(delimiter.join([str(DVAR.scale) for DVAR in self.dependentVariables.values()]))
+ prnt(
+ delimiter.join(
+ [str(DVAR.scale) for DVAR in self.dependentVariables.values()]
+ )
+ )
# Missing data indicators (This is -9999 (or -99999, etc.) for any missing data condition, except for the main time (independent) variable which is never missing) - comma delimited.
- prnt(delimiter.join([str(DVAR.miss)
- for DVAR in self.dependentVariables.values()]))
+ prnt(
+ delimiter.join(
+ [str(DVAR.miss) for DVAR in self.dependentVariables.values()]
+ )
+ )
# Variable names and units (Short variable name and units are required, and optional long descriptive name, in that order, and separated by commas. If the variable is unitless, enter the keyword "none" for its units. Each short variable name and units (and optional long name) are entered on one line. The short variable name must correspond exactly to the name used for that variable as a column header, i.e., the last header line prior to start of data.).
- _ = [prnt(DVAR.desc(delimiter))
- for DVAR in self.dependentVariables.values()]
+ _ = [prnt(DVAR.desc(delimiter)) for DVAR in self.dependentVariables.values()]
if self.format == Formats.FFI2110:
# Number of variables (Integer value showing the number of dependent variables: the total number of columns of data is this value plus one.).
prnt(len(self.auxiliaryVariables))
# Scale factors (1 for most cases, except where grossly inconvenient) - comma delimited.
- prnt(delimiter.join([str(AUXVAR.scale) for AUXVAR in self.auxiliaryVariables.values()]))
+ prnt(
+ delimiter.join(
+ [str(AUXVAR.scale) for AUXVAR in self.auxiliaryVariables.values()]
+ )
+ )
# Missing data indicators (This is -9999 (or -99999, etc.) for any missing data condition, except for the main time (independent) variable which is never missing) - comma delimited.
- prnt(delimiter.join([str(AUXVAR.miss)
- for AUXVAR in self.auxiliaryVariables.values()]))
+ prnt(
+ delimiter.join(
+ [str(AUXVAR.miss) for AUXVAR in self.auxiliaryVariables.values()]
+ )
+ )
# Variable names and units (Short variable name and units are required, and optional long descriptive name, in that order, and separated by commas. If the variable is unitless, enter the keyword "none" for its units. Each short variable name and units (and optional long name) are entered on one line. The short variable name must correspond exactly to the name used for that variable as a column header, i.e., the last header line prior to start of data.).
- _ = [prnt(AUXVAR.desc(delimiter))
- for AUXVAR in self.auxiliaryVariables.values()]
+ _ = [
+ prnt(AUXVAR.desc(delimiter))
+ for AUXVAR in self.auxiliaryVariables.values()
+ ]
# Number of SPECIAL comment lines (Integer value indicating the number of lines of special comments, NOT including this line.).
prnt("{:d}".format(len(self.specialComments)))
@@ -707,40 +818,48 @@ class Dataset:
# re-create last line out of actual data if missing...
if self.normalComments.shortnames == []:
self.normalComments.shortnames = delimiter.join(
- [self.variables[x].shortname for x in self.variables])
+ [self.variables[x].shortname for x in self.variables]
+ )
_ = [prnt(x) for x in self.normalComments]
- def writeData(self, f=sys.stdout, fmt=DEFAULT_NUM_FORMAT, delimiter=DEFAULT_FIELD_DELIM):
- '''Write data
+ def writeData(
+ self, f=sys.stdout, fmt=DEFAULT_NUM_FORMAT, delimiter=DEFAULT_FIELD_DELIM
+ ):
+ """Write data
:param f: handle to write to
:type f: file handle or StringIO stream, defaults to sys.stdout
- '''
+ """
self.data.write(f=f, fmt=fmt, delimiter=delimiter)
- def write(self, f=sys.stdout, fmt=DEFAULT_NUM_FORMAT, delimiter=DEFAULT_FIELD_DELIM):
- '''Write header and data
+ def write(
+ self, f=sys.stdout, fmt=DEFAULT_NUM_FORMAT, delimiter=DEFAULT_FIELD_DELIM
+ ):
+ """Write header and data
:param f: handle to write to
:type f: file handle or StringIO stream, defaults to sys.stdout
- '''
+ """
self.writeHeader(f=f, delimiter=delimiter)
self.writeData(f=f, fmt=fmt, delimiter=delimiter)
def endDefineMode(self):
- '''Fixes the variables structure of the dataset. Sets up the data store,
+ """Fixes the variables structure of the dataset. Sets up the data store,
so data can be added. Needs to be called after variable definition
and before adding data.
- '''
+ """
self.defineMode = False
# create data store
if self.format == Formats.FFI1001:
- self.data = DataStore1001(
- self.independentVariable, self.dependentVariables)
+ self.data = DataStore1001(self.independentVariable, self.dependentVariables)
elif self.format == Formats.FFI2110:
- self.data = DataStore2110(self.independentVariable, self.independentBoundedVariable,
- self.auxiliaryVariables, self.dependentVariables)
+ self.data = DataStore2110(
+ self.independentVariable,
+ self.independentBoundedVariable,
+ self.auxiliaryVariables,
+ self.dependentVariables,
+ )
def __del__(self):
try:
@@ -750,23 +869,22 @@ class Dataset:
pass
def __init__(self, f=None, loadData=True, splitChar=",", format=Formats.FFI1001):
- '''Constructor method
- '''
+ """Constructor method"""
self.format = format
self.version = None
- self.dataID = 'dataID'
- self.locationID = 'locationID'
+ self.dataID = "dataID"
+ self.locationID = "locationID"
self.revision = 0
self.launch = None
self.fileVolumeNumber = 1
self.totalNumberOfFileVolumes = 1
- self.PIName = 'Mustermann, Martin'
- self.PIAffiliation = 'Musterinstitut'
- self.dataSourceDescription = 'Musterdatenprodukt'
- self.missionName = 'MUSTEREX'
+ self.PIName = "Mustermann, Martin"
+ self.PIAffiliation = "Musterinstitut"
+ self.dataSourceDescription = "Musterdatenprodukt"
+ self.missionName = "MUSTEREX"
self.dateOfCollection = datetime.datetime.today()
self.dateOfRevision = datetime.datetime.today()
self.dataIntervalCode = [0.0]
@@ -790,11 +908,11 @@ class Dataset:
# read data if f is not None
if f is not None:
if isinstance(f, (str, pathlib.Path)):
- self.inputFhandle = open(f, 'r')
+ self.inputFhandle = open(f, "r")
else:
self.inputFhandle = f
self.readHeader(splitChar)
if loadData:
self.endDefineMode()
- self.readData(splitChar)
\ No newline at end of file
+ self.readData(splitChar)
diff --git a/tests/examples/AR_DC8_20050203_R0.ict b/tests/example_data/AR_DC8_20050203_R0.ict
similarity index 100%
rename from tests/examples/AR_DC8_20050203_R0.ict
rename to tests/example_data/AR_DC8_20050203_R0.ict
diff --git a/tests/examples/BB-FLUX_CU-SOF_20180808_R2.ict b/tests/example_data/BB-FLUX_CU-SOF_20180808_R2.ict
similarity index 100%
rename from tests/examples/BB-FLUX_CU-SOF_20180808_R2.ict
rename to tests/example_data/BB-FLUX_CU-SOF_20180808_R2.ict
diff --git a/tests/examples/DC8-20160517.ict b/tests/example_data/DC8-20160517.ict
similarity index 100%
rename from tests/examples/DC8-20160517.ict
rename to tests/example_data/DC8-20160517.ict
diff --git a/tests/examples/Dongdaemun_NIER_20160520_RA.ict b/tests/example_data/Dongdaemun_NIER_20160520_RA.ict
similarity index 100%
rename from tests/examples/Dongdaemun_NIER_20160520_RA.ict
rename to tests/example_data/Dongdaemun_NIER_20160520_RA.ict
diff --git a/tests/examples/HOX_DC8_20040712_R0.ict b/tests/example_data/HOX_DC8_20040712_R0.ict
similarity index 100%
rename from tests/examples/HOX_DC8_20040712_R0.ict
rename to tests/example_data/HOX_DC8_20040712_R0.ict
diff --git a/tests/examples/NOx_RHBrown_20040830_R0.ict b/tests/example_data/NOx_RHBrown_20040830_R0.ict
similarity index 100%
rename from tests/examples/NOx_RHBrown_20040830_R0.ict
rename to tests/example_data/NOx_RHBrown_20040830_R0.ict
diff --git a/tests/examples/PAVE-AR_DC8_20050203_R0.ict b/tests/example_data/PAVE-AR_DC8_20050203_R0.ict
similarity index 100%
rename from tests/examples/PAVE-AR_DC8_20050203_R0.ict
rename to tests/example_data/PAVE-AR_DC8_20050203_R0.ict
diff --git a/tests/examples/bt_Munich_2020061000_72.ict.txt b/tests/example_data/bt_Munich_2020061000_72.ict.txt
similarity index 100%
rename from tests/examples/bt_Munich_2020061000_72.ict.txt
rename to tests/example_data/bt_Munich_2020061000_72.ict.txt
diff --git a/tests/examples/korusaq-flexpart-dc8_trajectory_20160529_R2.ict b/tests/example_data/korusaq-flexpart-dc8_trajectory_20160529_R2.ict
similarity index 100%
rename from tests/examples/korusaq-flexpart-dc8_trajectory_20160529_R2.ict
rename to tests/example_data/korusaq-flexpart-dc8_trajectory_20160529_R2.ict
diff --git a/tests/examples/korusaq-mrg01-HANSEO-KING-AIR_merge_20160507_RA.ict b/tests/example_data/korusaq-mrg01-HANSEO-KING-AIR_merge_20160507_RA.ict
similarity index 100%
rename from tests/examples/korusaq-mrg01-HANSEO-KING-AIR_merge_20160507_RA.ict
rename to tests/example_data/korusaq-mrg01-HANSEO-KING-AIR_merge_20160507_RA.ict
diff --git a/tests/examples/korusaq-mrg10-dc8_merge_20160510_R4.ict b/tests/example_data/korusaq-mrg10-dc8_merge_20160510_R4.ict
similarity index 100%
rename from tests/examples/korusaq-mrg10-dc8_merge_20160510_R4.ict
rename to tests/example_data/korusaq-mrg10-dc8_merge_20160510_R4.ict
diff --git a/tests/examples/read_and_write.py b/tests/example_data/read_and_write.py
similarity index 100%
rename from tests/examples/read_and_write.py
rename to tests/example_data/read_and_write.py
diff --git a/tests/examples/will_fail/AROTAL-RAY_DC8_20040715_R1.ict b/tests/example_data/will_fail/AROTAL-RAY_DC8_20040715_R1.ict
similarity index 100%
rename from tests/examples/will_fail/AROTAL-RAY_DC8_20040715_R1.ict
rename to tests/example_data/will_fail/AROTAL-RAY_DC8_20040715_R1.ict
diff --git a/tests/examples/will_fail/LIDARO3_WP3_20040830_R0.ict b/tests/example_data/will_fail/LIDARO3_WP3_20040830_R0.ict
similarity index 100%
rename from tests/examples/will_fail/LIDARO3_WP3_20040830_R0.ict
rename to tests/example_data/will_fail/LIDARO3_WP3_20040830_R0.ict
diff --git a/tests/test_dataset.py b/tests/test_dataset.py
index 10e2085..3da2f3e 100644
--- a/tests/test_dataset.py
+++ b/tests/test_dataset.py
@@ -7,7 +7,7 @@ import icartt
# working directory, example files
wd = pathlib.Path(__file__).parent
-fns = (wd / "examples").glob("*.ict")
+fns = (wd / "example_data").glob("*.ict")
def compareFiles(fn, strIn, strOut, skiplines=0, nlines=-1): # pragma: no cover
@@ -70,7 +70,7 @@ def compareFiles(fn, strIn, strOut, skiplines=0, nlines=-1): # pragma: no cover
class Simple1001TestCase(unittest.TestCase):
def setUp(self):
- self.fn = wd / "examples/NOx_RHBrown_20040830_R0.ict"
+ self.fn = wd / "example_data" / "NOx_RHBrown_20040830_R0.ict"
self.nHeader = 41
def tearDown(self):
@@ -309,10 +309,6 @@ class Create1001TestCase(unittest.TestCase):
return True
-# fns = [ os.path.join("tests", "examples", fn) for fn in os.listdir(os.path.join("tests", "examples")) if fn.endswith(".ict")]
-# fns = [ "tests/examples/AROTAL-RAY_DC8_20040715_R1.ict" ]
-
-
class BulkIOTestCase(unittest.TestCase):
def testOpen(self):
for fn in fns:
diff --git a/tests/usage_examples/create_ffi1001.py b/tests/usage_examples/create_ffi1001.py
new file mode 100644
index 0000000..21e4b8c
--- /dev/null
+++ b/tests/usage_examples/create_ffi1001.py
@@ -0,0 +1,72 @@
+
+import icartt
+import datetime
+
+ict = icartt.Dataset(format=icartt.Formats.FFI1001)
+
+ict.PIName = 'Knote, Christoph'
+ict.PIAffiliation = 'Faculty of Medicine, University Augsburg, Germany'
+ict.dataSourceDescription = 'Example data'
+ict.missionName = 'MBEES'
+ict.dateOfCollection = datetime.datetime.today()
+ict.dateOfRevision = datetime.datetime.today()
+
+ict.dataIntervalCode = [ 0 ]
+
+ict.independentVariable = icartt.Variable( 'Time_Start',
+ 'seconds_from_0_hours_on_valid_date',
+ 'Time_Start',
+ 'Time_Start',
+ vartype=icartt.VariableType.IndependentVariable,
+ scale=1.0, miss=-9999999)
+
+ict.dependentVariables['Time_Stop'] = icartt.Variable( 'Time_Stop',
+ 'seconds_from_0_hours_on_valid_date',
+ 'Time_Stop',
+ 'Time_Stop',
+ scale=1.0, miss=-9999999)
+
+ict.dependentVariables['Payload'] = icartt.Variable( 'Payload',
+ 'some_units',
+ 'Payload',
+ 'Payload',
+ scale=1.0, miss=-9999999)
+
+ict.specialComments.append("Some comments on this dataset:")
+ict.specialComments.append("They are just examples!")
+ict.specialComments.append("Adapt as needed.")
+
+ict.endDefineMode()
+
+# Three ways to add data:
+
+# 1) simple (single data line)
+ict.data.add( Time_Start = 12.3, Time_Stop = 12.5, Payload = 23789423.2e5 )
+
+# Let's check:
+ict.write()
+
+# 2) as dictionary (single data line)
+mydict = { 'Time_Start': 12.6, 'Time_Stop': 13.1, 'Payload': 324235644.1e5 }
+ict.data.add( **mydict )
+# (note, exploding the dictionary is necessary)
+
+# 3) as NumPy array (bulk)
+import numpy as np
+data = np.array( [ (13.4, 14.0, 2348925e5), (14.1, 14.9, 23425634e5) ] )
+ict.data.addBulk( data )
+
+# Note 1: you are responsible to ensure that the order of elements in a data line
+# corresponds to variable listing below:
+print( [ x for x in ict.variables ] )
+
+# Note 2: for single lines, you still need to make it an array!
+data = np.array( [ (15.4, 15.0, 52452495290e5) ] )
+ict.data.addBulk( data )
+
+# Now, look at it in ICARTT form:
+ict.write()
+
+# And you could simply write to file:
+#with open('output.ict', 'w') as f:
+# ict.write(f=f)
\ No newline at end of file
diff --git a/tests/usage_examples/create_ffi2110.py b/tests/usage_examples/create_ffi2110.py
new file mode 100644
index 0000000..8796a22
--- /dev/null
+++ b/tests/usage_examples/create_ffi2110.py
@@ -0,0 +1,104 @@
+import icartt
+import datetime
+import numpy as np
+
+ict = icartt.Dataset(format=icartt.Formats.FFI2110)
+
+ict.PIName = 'Knote, Christoph'
+ict.PIAffiliation = 'Faculty of Medicine, University Augsburg, Germany'
+ict.dataSourceDescription = 'Example data'
+ict.missionName = 'MBEES'
+ict.dateOfCollection = datetime.datetime.today()
+ict.dateOfRevision = datetime.datetime.today()
+
+ict.dataIntervalCode = [ 0 ]
+
+ict.independentVariable = icartt.Variable( 'Time_Start',
+ 'seconds_from_0_hours_on_valid_date',
+ 'Time_Start',
+ 'Time_Start',
+ vartype=icartt.VariableType.IndependentVariable,
+ scale=1.0, miss=-9999999)
+
+ict.independentBoundedVariable = icartt.Variable( 'Altitude',
+ 'altitude_above_ground_in_meters',
+ 'Altitude',
+ 'Altitude',
+ vartype=icartt.VariableType.IndependentBoundedVariable,
+ scale=1.0, miss=-9999999)
+
+# ICARTT convention: first aux variable contains number of dependent elements
+ict.auxiliaryVariables['nAltitudes'] = icartt.Variable( 'nAltitudes',
+ 'number_of_dependent_variable_items',
+ 'variable',
+ 'nAltitudes',
+ scale=1.0, miss=-9999999)
+
+ict.auxiliaryVariables['Time_Stop'] = icartt.Variable( 'Time_Stop',
+ 'seconds_from_0_hours_on_valid_date',
+ 'Time_Stop',
+ 'Time_Stop',
+ scale=1.0, miss=-9999999)
+
+ict.auxiliaryVariables['Longitude'] = icartt.Variable( 'Longitude',
+ 'longitude_in_degrees',
+ 'Longitude',
+ 'Longitude',
+ scale=1.0, miss=-9999999)
+
+ict.auxiliaryVariables['Latitude'] = icartt.Variable( 'Latitude',
+ 'latitude_in_degrees',
+ 'Latitude',
+ 'Latitude',
+ scale=1.0, miss=-9999999)
+
+ict.dependentVariables['Payload1'] = icartt.Variable( 'Payload1',
+ 'some_units',
+ 'Payload1',
+ 'Payload1',
+ scale=1.0, miss=-9999999)
+
+ict.dependentVariables['Payload2'] = icartt.Variable( 'Payload2',
+ 'some_units',
+ 'Payload2',
+ 'Payload2',
+ scale=1.0, miss=-9999999)
+
+ict.specialComments.append("Some comments on this dataset:")
+ict.specialComments.append("They are just examples!")
+ict.specialComments.append("Adapt as needed.")
+
+ict.endDefineMode()
+
+# Add data
+
+# the three ways to add data (see FFI 1001) are still possible for FFI 2110
+
+# a new independent variable item is created by adding data for the new item
+# with its auxiliary data information:
+
+ict.data.add( Time_Start = 12.3, nAuxiliary=4, Time_Stop = 12.5, Latitude = 48.21, Longitude = 10.3 )
+ict.data.add( Time_Start = 13.3, nAuxiliary=2, Time_Stop = 13.5, Latitude = 48.31, Longitude = 10.4 )
+
+# then, dependent data can be added:
+
+# ibvar, dvar1, dvar2
+data = np.array( [ ( 0, 123, 8.4e4),
+ (100, 122, 9.1e4),
+ (250, 115, 9.3e4),
+ (500, 106, 9.8e4) ] )
+
+ict.data.addBulkDep(12.3, data)
+
+# ibvar, dvar1, dvar2
+data = np.array( [ ( 0, 153, 7.3e4),
+ (270, 172, 8.9e4) ] )
+
+ict.data.addBulkDep(13.3, data)
+
+# Now, look at it in ICARTT form:
+ict.write()
+
+# And you could simply write to file:
+#with open('output.ict', 'w') as f:
+# ict.write(f=f)
diff --git a/tests/usage_examples/read_ffi1001.py b/tests/usage_examples/read_ffi1001.py
new file mode 100644
index 0000000..f08e08e
--- /dev/null
+++ b/tests/usage_examples/read_ffi1001.py
@@ -0,0 +1,28 @@
+import icartt
+import pathlib
+
+# load a new dataset from an existing file
+wd = pathlib.Path(__file__).parent
+ict = icartt.Dataset( wd / ".." / "example_data" / 'DC8-20160517.ict')
+
+# read some metadata
+ict.PIName
+ict.PIAffiliation
+ict.missionName
+ict.dataSourceDescription
+
+# list variable names
+[ x for x in ict.variables ]
+
+# some info on a variable
+ict.variables['Alt_ft'].units
+ict.variables['Alt_ft'].miss
+
+# get data for variable 'UTC':
+ict.data['UTC']
+
+# get all data (NumPy array):
+ict.data[:]
+
+# get the altitude in feet for those data where UTC < 86400.0:
+ict.data[ ict.data['UTC'] < 86400.0 ]['Alt_ft']
diff --git a/tests/usage_examples/read_ffi2110.py b/tests/usage_examples/read_ffi2110.py
new file mode 100644
index 0000000..4b19101
--- /dev/null
+++ b/tests/usage_examples/read_ffi2110.py
@@ -0,0 +1,34 @@
+import icartt
+import pathlib
+
+# load a new dataset from an existing file
+wd = pathlib.Path(__file__).parent
+ict = icartt.Dataset( wd / ".." / "example_data" / 'AR_DC8_20050203_R0.ict')
+
+# list variable names
+[ x for x in ict.variables ]
+
+# independent, independent bounded, dependent, auxiliary variables?
+print(ict.independentVariable.shortname)
+print(ict.independentBoundedVariable.shortname)
+print([ x for x in ict.auxiliaryVariables])
+print([ x for x in ict.dependentVariables])
+
+# some info on a variable
+ict.variables['Latitude'].units
+ict.variables['Latitude'].miss
+
+# get steps for which data is available:
+tsteps = [ x for x in ict.data ]
+
+# let's look at the first time step data
+ict.data[ tsteps[0] ]
+
+# auxiliary data at this time step:
+ict.data[ tsteps[0] ]['AUX'][:]
+
+# dependent data at this time step:
+tstepdata = ict.data[ tsteps[0] ]['DEP'][:]
+
+# get the ozone mixing ratio for those data where Altitude < 10000.0:
+print(tstepdata[ tstepdata['Altitude[]'] < 10000.0 ]['O3_MR[]'])
--
GitLab