From b2aa09bce17a5e561dbf8ee0ca8fe2addab7fe24 Mon Sep 17 00:00:00 2001 From: qian-chu Date: Mon, 10 Jun 2024 21:57:28 +0200 Subject: [PATCH 01/12] correct annotation onset during exportation --- mne/export/_edf.py | 4 +- mne/export/_eeglab.py | 4 +- mne/export/tests/test_export.py | 73 +++++++++++++++++++++++++++++---- 3 files changed, 72 insertions(+), 9 deletions(-) diff --git a/mne/export/_edf.py b/mne/export/_edf.py index 3f7e55b3d77..6c1a207638e 100644 --- a/mne/export/_edf.py +++ b/mne/export/_edf.py @@ -200,7 +200,9 @@ def _export_raw(fname, raw, physical_range, add_ch_type): annotations = [] for desc, onset, duration, ch_names in zip( raw.annotations.description, - raw.annotations.onset, + # subtract raw.first_time because EDF marks events starting from the first + # available data point and ignores raw.first_time + raw.annotations.onset - raw.first_time, raw.annotations.duration, raw.annotations.ch_names, ): diff --git a/mne/export/_eeglab.py b/mne/export/_eeglab.py index f8095cfc4f0..b03892d7105 100644 --- a/mne/export/_eeglab.py +++ b/mne/export/_eeglab.py @@ -27,7 +27,9 @@ def _export_raw(fname, raw): annotations = [ raw.annotations.description, - raw.annotations.onset, + # subtract raw.first_time because EEGLAB marks events starting from the first + # available data point and ignores raw.first_time + raw.annotations.onset - raw.first_time, raw.annotations.duration, ] eeglabio.raw.export_set( diff --git a/mne/export/tests/test_export.py b/mne/export/tests/test_export.py index 9c8a60f50bb..d0dc955376c 100644 --- a/mne/export/tests/test_export.py +++ b/mne/export/tests/test_export.py @@ -121,6 +121,42 @@ def test_export_raw_eeglab(tmp_path): with pytest.warns(RuntimeWarning, match="Raw instance has unapplied projectors."): raw.export(temp_fname, overwrite=True) +@pytest.mark.parametrize("tmin", (0, 1, 5, 10)) +def test_export_raw_eeglab_annotations(tmp_path, tmin): + """Test that exporting EEGLAB preserves annotations and corects for raw.first_time.""" + pytest.importorskip("eeglabio") + raw = read_raw_fif(fname_raw, preload=True) + raw.apply_proj() + annotations = Annotations( + onset=[0.01, 0.05, 0.90, 1.05], + duration=[0, 1, 0, 0], + description=["test1", "test2", "test3", "test4"], + ch_names=[["MEG 0113"], ["MEG 0113", "MEG 0132"], [], ["MEG 0143"]], + ) + raw.set_annotations(annotations) + raw.crop(tmin) + + # export + temp_fname = tmp_path / "test.set" + raw.export(temp_fname) + + # read in the file + with pytest.warns(RuntimeWarning, match="is above the 99th percentile"): + raw_read = read_raw_eeglab(temp_fname, preload=True, montage_units="m") + assert raw_read.first_time == 0 + + valid_annot = raw.annotations.onset >= tmin + assert_array_almost_equal( + raw.annotations.onset[valid_annot] - raw.first_time, + raw_read.annotations.onset - raw_read.first_time, + ) + assert_array_equal( + raw.annotations.duration[valid_annot], raw_read.annotations.duration + ) + assert_array_equal( + raw.annotations.description[valid_annot], raw_read.annotations.description + ) + def _create_raw_for_edf_tests(stim_channel_index=None): rng = np.random.RandomState(12345) @@ -217,8 +253,9 @@ def test_edf_physical_range(tmp_path): @edfio_mark() -def test_export_edf_annotations(tmp_path): - """Test that exporting EDF preserves annotations.""" +@pytest.mark.parametrize("tmin", (0, 0.005, 0.03, 1)) +def test_export_edf_annotations(tmp_path, tmin): + """Test that exporting EDF preserves annotations and corects for raw.first_time.""" raw = _create_raw_for_edf_tests() annotations = Annotations( onset=[0.01, 0.05, 0.90, 1.05], @@ -227,17 +264,39 @@ def test_export_edf_annotations(tmp_path): ch_names=[["0"], ["0", "1"], [], ["1"]], ) raw.set_annotations(annotations) + raw.crop(tmin) + assert raw.first_time == tmin + + if tmin % 1 == 0: + expectation = nullcontext() + else: + expectation = pytest.warns( + RuntimeWarning, match="EDF format requires equal-length data blocks" + ) # export temp_fname = tmp_path / "test.edf" - raw.export(temp_fname) + with expectation: + raw.export(temp_fname, physical_range=(0, 10)) # read in the file raw_read = read_raw_edf(temp_fname, preload=True) - assert_array_equal(raw.annotations.onset, raw_read.annotations.onset) - assert_array_equal(raw.annotations.duration, raw_read.annotations.duration) - assert_array_equal(raw.annotations.description, raw_read.annotations.description) - assert_array_equal(raw.annotations.ch_names, raw_read.annotations.ch_names) + assert raw_read.first_time == 0 + + valid_annot = raw.annotations.onset >= tmin + assert_array_almost_equal( + raw.annotations.onset[valid_annot] - raw.first_time, + raw_read.annotations.onset - raw_read.first_time, + ) + assert_array_equal( + raw.annotations.duration[valid_annot], raw_read.annotations.duration + ) + assert_array_equal( + raw.annotations.description[valid_annot], raw_read.annotations.description + ) + assert_array_equal( + raw.annotations.ch_names[valid_annot], raw_read.annotations.ch_names + ) @edfio_mark() From 1643b3c8167fbeb9d843eb5de344cdd6e6b251e1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 20:00:14 +0000 Subject: [PATCH 02/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mne/export/tests/test_export.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mne/export/tests/test_export.py b/mne/export/tests/test_export.py index d0dc955376c..f76325f25b8 100644 --- a/mne/export/tests/test_export.py +++ b/mne/export/tests/test_export.py @@ -121,6 +121,7 @@ def test_export_raw_eeglab(tmp_path): with pytest.warns(RuntimeWarning, match="Raw instance has unapplied projectors."): raw.export(temp_fname, overwrite=True) + @pytest.mark.parametrize("tmin", (0, 1, 5, 10)) def test_export_raw_eeglab_annotations(tmp_path, tmin): """Test that exporting EEGLAB preserves annotations and corects for raw.first_time.""" From c8378524d3b629b8e478082fb4a06490b028cde8 Mon Sep 17 00:00:00 2001 From: qian-chu Date: Mon, 10 Jun 2024 22:10:21 +0200 Subject: [PATCH 03/12] Create 12656.bugfix.rst --- doc/changes/devel/12656.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/changes/devel/12656.bugfix.rst diff --git a/doc/changes/devel/12656.bugfix.rst b/doc/changes/devel/12656.bugfix.rst new file mode 100644 index 00000000000..b3a0c62539a --- /dev/null +++ b/doc/changes/devel/12656.bugfix.rst @@ -0,0 +1 @@ +Fix bug where :func:`mne.export.export_raw` does not correct for recording start time (`raw.first_time`) when exporting Raw instances to EDF or EEGLAB formats, by `Qian Chu`_. \ No newline at end of file From bbe650158b8a6952cd8ad029f1ea522fda84de1f Mon Sep 17 00:00:00 2001 From: qian-chu Date: Mon, 10 Jun 2024 22:47:07 +0200 Subject: [PATCH 04/12] formatting --- mne/export/tests/test_export.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mne/export/tests/test_export.py b/mne/export/tests/test_export.py index f76325f25b8..89e50c79def 100644 --- a/mne/export/tests/test_export.py +++ b/mne/export/tests/test_export.py @@ -124,7 +124,8 @@ def test_export_raw_eeglab(tmp_path): @pytest.mark.parametrize("tmin", (0, 1, 5, 10)) def test_export_raw_eeglab_annotations(tmp_path, tmin): - """Test that exporting EEGLAB preserves annotations and corects for raw.first_time.""" + """Test that exporting EEGLAB preserves annotations + and corrects for raw.first_time.""" pytest.importorskip("eeglabio") raw = read_raw_fif(fname_raw, preload=True) raw.apply_proj() @@ -256,7 +257,8 @@ def test_edf_physical_range(tmp_path): @edfio_mark() @pytest.mark.parametrize("tmin", (0, 0.005, 0.03, 1)) def test_export_edf_annotations(tmp_path, tmin): - """Test that exporting EDF preserves annotations and corects for raw.first_time.""" + """Test that exporting EDF preserves annotations + and corrects for raw.first_time.""" raw = _create_raw_for_edf_tests() annotations = Annotations( onset=[0.01, 0.05, 0.90, 1.05], From 1eda9731b06c9f0fd5f45031b00a6df377386c22 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 20:47:24 +0000 Subject: [PATCH 05/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mne/export/tests/test_export.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mne/export/tests/test_export.py b/mne/export/tests/test_export.py index 89e50c79def..c22fc7f25ae 100644 --- a/mne/export/tests/test_export.py +++ b/mne/export/tests/test_export.py @@ -125,7 +125,8 @@ def test_export_raw_eeglab(tmp_path): @pytest.mark.parametrize("tmin", (0, 1, 5, 10)) def test_export_raw_eeglab_annotations(tmp_path, tmin): """Test that exporting EEGLAB preserves annotations - and corrects for raw.first_time.""" + and corrects for raw.first_time. + """ pytest.importorskip("eeglabio") raw = read_raw_fif(fname_raw, preload=True) raw.apply_proj() @@ -258,7 +259,8 @@ def test_edf_physical_range(tmp_path): @pytest.mark.parametrize("tmin", (0, 0.005, 0.03, 1)) def test_export_edf_annotations(tmp_path, tmin): """Test that exporting EDF preserves annotations - and corrects for raw.first_time.""" + and corrects for raw.first_time. + """ raw = _create_raw_for_edf_tests() annotations = Annotations( onset=[0.01, 0.05, 0.90, 1.05], From 78eb8ac6f8d116679ae1e1ca4d75cda1dc879207 Mon Sep 17 00:00:00 2001 From: qian-chu Date: Mon, 10 Jun 2024 22:54:00 +0200 Subject: [PATCH 06/12] formatting func desc --- mne/export/tests/test_export.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mne/export/tests/test_export.py b/mne/export/tests/test_export.py index c22fc7f25ae..ecd32559e64 100644 --- a/mne/export/tests/test_export.py +++ b/mne/export/tests/test_export.py @@ -124,7 +124,8 @@ def test_export_raw_eeglab(tmp_path): @pytest.mark.parametrize("tmin", (0, 1, 5, 10)) def test_export_raw_eeglab_annotations(tmp_path, tmin): - """Test that exporting EEGLAB preserves annotations + """ + Test that exporting EEGLAB preserves annotations and corrects for raw.first_time. """ pytest.importorskip("eeglabio") @@ -258,7 +259,8 @@ def test_edf_physical_range(tmp_path): @edfio_mark() @pytest.mark.parametrize("tmin", (0, 0.005, 0.03, 1)) def test_export_edf_annotations(tmp_path, tmin): - """Test that exporting EDF preserves annotations + """ + Test that exporting EDF preserves annotations and corrects for raw.first_time. """ raw = _create_raw_for_edf_tests() From 590cf165e4bc928693ef6311aa480dbc486597d8 Mon Sep 17 00:00:00 2001 From: qian-chu Date: Mon, 10 Jun 2024 22:58:21 +0200 Subject: [PATCH 07/12] summary line and desc separation --- mne/export/tests/test_export.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mne/export/tests/test_export.py b/mne/export/tests/test_export.py index ecd32559e64..8ab95d68687 100644 --- a/mne/export/tests/test_export.py +++ b/mne/export/tests/test_export.py @@ -124,9 +124,9 @@ def test_export_raw_eeglab(tmp_path): @pytest.mark.parametrize("tmin", (0, 1, 5, 10)) def test_export_raw_eeglab_annotations(tmp_path, tmin): - """ - Test that exporting EEGLAB preserves annotations - and corrects for raw.first_time. + """Test annotations in the exported EEGLAB file + + All annotations should be perserved and onset corrected """ pytest.importorskip("eeglabio") raw = read_raw_fif(fname_raw, preload=True) @@ -259,9 +259,9 @@ def test_edf_physical_range(tmp_path): @edfio_mark() @pytest.mark.parametrize("tmin", (0, 0.005, 0.03, 1)) def test_export_edf_annotations(tmp_path, tmin): - """ - Test that exporting EDF preserves annotations - and corrects for raw.first_time. + """Test annotations in the exported EDF file + + All annotations should be perserved and onset corrected """ raw = _create_raw_for_edf_tests() annotations = Annotations( From 970303c96576250fe61fe515b0b369674b8fa34b Mon Sep 17 00:00:00 2001 From: qian-chu Date: Mon, 10 Jun 2024 23:00:26 +0200 Subject: [PATCH 08/12] fix typo --- mne/export/tests/test_export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mne/export/tests/test_export.py b/mne/export/tests/test_export.py index 8ab95d68687..3d8cfcd21ac 100644 --- a/mne/export/tests/test_export.py +++ b/mne/export/tests/test_export.py @@ -126,7 +126,7 @@ def test_export_raw_eeglab(tmp_path): def test_export_raw_eeglab_annotations(tmp_path, tmin): """Test annotations in the exported EEGLAB file - All annotations should be perserved and onset corrected + All annotations should be preserved and onset corrected """ pytest.importorskip("eeglabio") raw = read_raw_fif(fname_raw, preload=True) @@ -261,7 +261,7 @@ def test_edf_physical_range(tmp_path): def test_export_edf_annotations(tmp_path, tmin): """Test annotations in the exported EDF file - All annotations should be perserved and onset corrected + All annotations should be preserved and onset corrected """ raw = _create_raw_for_edf_tests() annotations = Annotations( From cc96f10497c4a5e23d2c932c305497cfe09fbc8b Mon Sep 17 00:00:00 2001 From: qian-chu Date: Mon, 10 Jun 2024 23:01:58 +0200 Subject: [PATCH 09/12] add period --- mne/export/tests/test_export.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mne/export/tests/test_export.py b/mne/export/tests/test_export.py index 3d8cfcd21ac..ce71205f557 100644 --- a/mne/export/tests/test_export.py +++ b/mne/export/tests/test_export.py @@ -124,9 +124,9 @@ def test_export_raw_eeglab(tmp_path): @pytest.mark.parametrize("tmin", (0, 1, 5, 10)) def test_export_raw_eeglab_annotations(tmp_path, tmin): - """Test annotations in the exported EEGLAB file + """Test annotations in the exported EEGLAB file. - All annotations should be preserved and onset corrected + All annotations should be preserved and onset corrected. """ pytest.importorskip("eeglabio") raw = read_raw_fif(fname_raw, preload=True) @@ -259,9 +259,9 @@ def test_edf_physical_range(tmp_path): @edfio_mark() @pytest.mark.parametrize("tmin", (0, 0.005, 0.03, 1)) def test_export_edf_annotations(tmp_path, tmin): - """Test annotations in the exported EDF file + """Test annotations in the exported EDF file. - All annotations should be preserved and onset corrected + All annotations should be preserved and onset corrected. """ raw = _create_raw_for_edf_tests() annotations = Annotations( From f8f118a97cffa9cdd5d3fdbcfeb5d5f5081c57e3 Mon Sep 17 00:00:00 2001 From: qian-chu <97355086+qian-chu@users.noreply.github.com> Date: Sat, 24 Aug 2024 22:34:14 +0200 Subject: [PATCH 10/12] use _sync_onset --- mne/export/_edf.py | 3 ++- mne/export/_eeglab.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mne/export/_edf.py b/mne/export/_edf.py index 1e7d7339392..ef53ba41ccd 100644 --- a/mne/export/_edf.py +++ b/mne/export/_edf.py @@ -7,6 +7,7 @@ import numpy as np +from ..annotations import _sync_onset from ..utils import _check_edfio_installed, warn _check_edfio_installed() @@ -204,7 +205,7 @@ def _export_raw(fname, raw, physical_range, add_ch_type): raw.annotations.description, # subtract raw.first_time because EDF marks events starting from the first # available data point and ignores raw.first_time - raw.annotations.onset - raw.first_time, + _sync_onset(raw, raw.annotations.onset, inverse=False), raw.annotations.duration, raw.annotations.ch_names, ): diff --git a/mne/export/_eeglab.py b/mne/export/_eeglab.py index b6d67653fca..7ed6870db09 100644 --- a/mne/export/_eeglab.py +++ b/mne/export/_eeglab.py @@ -4,6 +4,7 @@ import numpy as np +from ..annotations import _sync_onset from ..utils import _check_eeglabio_installed _check_eeglabio_installed() @@ -28,7 +29,7 @@ def _export_raw(fname, raw): raw.annotations.description, # subtract raw.first_time because EEGLAB marks events starting from the first # available data point and ignores raw.first_time - raw.annotations.onset - raw.first_time, + _sync_onset(raw, raw.annotations.onset, inverse=False), raw.annotations.duration, ] eeglabio.raw.export_set( From e556b961d8cff55716a1db8c18a2a821d281b865 Mon Sep 17 00:00:00 2001 From: qian-chu Date: Fri, 13 Dec 2024 21:22:31 +0100 Subject: [PATCH 11/12] correct tests --- mne/export/tests/test_export.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/mne/export/tests/test_export.py b/mne/export/tests/test_export.py index 663d75e8eda..f0c6dc5a929 100644 --- a/mne/export/tests/test_export.py +++ b/mne/export/tests/test_export.py @@ -147,12 +147,13 @@ def test_export_raw_eeglab_annotations(tmp_path, tmin): # read in the file with pytest.warns(RuntimeWarning, match="is above the 99th percentile"): raw_read = read_raw_eeglab(temp_fname, preload=True, montage_units="m") - assert raw_read.first_time == 0 - - valid_annot = raw.annotations.onset >= tmin + assert raw_read.first_time == 0 # exportation resets first_time + valid_annot = raw.annotations.onset >= tmin # only annotations in the cropped range gets exported + + # compare annotations before and after export assert_array_almost_equal( raw.annotations.onset[valid_annot] - raw.first_time, - raw_read.annotations.onset - raw_read.first_time, + raw_read.annotations.onset, ) assert_array_equal( raw.annotations.duration[valid_annot], raw_read.annotations.duration @@ -194,6 +195,7 @@ def test_double_export_edf(tmp_path): """Test exporting an EDF file multiple times.""" raw = _create_raw_for_edf_tests(stim_channel_index=2) raw.info.set_meas_date("2023-09-04 14:53:09.000") + raw.set_annotations(Annotations(onset=[1], duration=[0], description=["test"])) # include subject info and measurement date raw.info["subject_info"] = dict( @@ -315,7 +317,7 @@ def test_export_edf_annotations(tmp_path, tmin): raw.crop(tmin) assert raw.first_time == tmin - if tmin % 1 == 0: + if raw.n_times % raw.info["sfreq"] == 0: expectation = nullcontext() else: expectation = pytest.warns( @@ -325,16 +327,20 @@ def test_export_edf_annotations(tmp_path, tmin): # export temp_fname = tmp_path / "test.edf" with expectation: - raw.export(temp_fname, physical_range=(0, 10)) + raw.export(temp_fname) # read in the file raw_read = read_raw_edf(temp_fname, preload=True) - assert raw_read.first_time == 0 - - valid_annot = raw.annotations.onset >= tmin + assert raw_read.first_time == 0 # exportation resets first_time + bad_annot = raw_read.annotations.description == "BAD_ACQ_SKIP" + if bad_annot.any(): + raw_read.annotations.delete(bad_annot) + valid_annot = raw.annotations.onset >= tmin # only annotations in the cropped range gets exported + + # compare annotations before and after export assert_array_almost_equal( raw.annotations.onset[valid_annot] - raw.first_time, - raw_read.annotations.onset - raw_read.first_time, + raw_read.annotations.onset ) assert_array_equal( raw.annotations.duration[valid_annot], raw_read.annotations.duration From ab728d3546e7118808b054695aa55101ae1ff4b7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 20:22:53 +0000 Subject: [PATCH 12/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mne/export/tests/test_export.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/mne/export/tests/test_export.py b/mne/export/tests/test_export.py index f0c6dc5a929..29e95034064 100644 --- a/mne/export/tests/test_export.py +++ b/mne/export/tests/test_export.py @@ -147,9 +147,11 @@ def test_export_raw_eeglab_annotations(tmp_path, tmin): # read in the file with pytest.warns(RuntimeWarning, match="is above the 99th percentile"): raw_read = read_raw_eeglab(temp_fname, preload=True, montage_units="m") - assert raw_read.first_time == 0 # exportation resets first_time - valid_annot = raw.annotations.onset >= tmin # only annotations in the cropped range gets exported - + assert raw_read.first_time == 0 # exportation resets first_time + valid_annot = ( + raw.annotations.onset >= tmin + ) # only annotations in the cropped range gets exported + # compare annotations before and after export assert_array_almost_equal( raw.annotations.onset[valid_annot] - raw.first_time, @@ -331,16 +333,17 @@ def test_export_edf_annotations(tmp_path, tmin): # read in the file raw_read = read_raw_edf(temp_fname, preload=True) - assert raw_read.first_time == 0 # exportation resets first_time + assert raw_read.first_time == 0 # exportation resets first_time bad_annot = raw_read.annotations.description == "BAD_ACQ_SKIP" if bad_annot.any(): raw_read.annotations.delete(bad_annot) - valid_annot = raw.annotations.onset >= tmin # only annotations in the cropped range gets exported - + valid_annot = ( + raw.annotations.onset >= tmin + ) # only annotations in the cropped range gets exported + # compare annotations before and after export assert_array_almost_equal( - raw.annotations.onset[valid_annot] - raw.first_time, - raw_read.annotations.onset + raw.annotations.onset[valid_annot] - raw.first_time, raw_read.annotations.onset ) assert_array_equal( raw.annotations.duration[valid_annot], raw_read.annotations.duration