Tests load_for_restart, next_path + force_suffix

- Added slow tests for load_for_restart function - Function next_path moves to a module snek5000.util.files and gets a force_suffix parameter
snek5000 · Oct 27, 2021 · adc45e1 · adc45e1
1 parent 216385d
commit adc45e1
Show file tree

Hide file tree

Showing 11 changed files with 203 additions and 33 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -58,7 +58,8 @@ jobs:
     - name: Run tests
       run: |
         source activate.sh
-        pytest -s --cov-report=xml --cov-report=term-missing
+        export SNEK_DEBUG=1
+        pytest -s --runslow --cov-report=xml --cov-report=term-missing
 
     - name: Upload coverage to codecov
       if: ${{ success() }}

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -29,12 +29,18 @@ Security in case of vulnerabilities.
 
 ## [Unreleased]
 
+### Added
+
+- {func}`snek5000.util.restart.load_for_restart` function
+- {mod}`snek5000.util.files`
+
 ### Changed
 
 - `params.oper.elem.staggered` has a new default value `auto`, which sets
   staggered grid if a linear solver is used and a collocated one if some other
   solver is used. Explicitly setting `params.oper.elem.staggered = True` is
   required to maintain previous default behaviour.
+- {func}`snek5000.util.files.next_path` gets a `force_suffix` parameter
 
 ### Fixed
 

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -62,6 +62,8 @@ pip install "phill @ git+https://github.com/exabl/snek5000-phill.git"
 
 * **Testing**: [Run `pytest`](https://pytest.readthedocs.io/) from the
   top-level directory. The test-cases can be found under `tests/` directory.
+  To run the slow tests too execute `pytest --runslow`.
+
 * **Debugging**: Set the environment variable:
   ```bash
   export SNEK_DEBUG=true

diff --git a/pyproject.toml b/pyproject.toml
@@ -7,7 +7,7 @@ write_to = "src/snek5000/_version.py"
 write_to_template = "__version__ = \"{version}\"\n"
 
 [tool.pytest.ini_options]
-addopts = "--cov=snek5000 --cov=tests"
+addopts = "--cov=snek5000 --cov=tests --no-cov-on-fail"
 
 [tool.isort]
 known_first_party = ["fluiddyn", "fluidsim", "snek5000"]

diff --git a/setup.cfg b/setup.cfg
@@ -55,6 +55,7 @@ tests =
     pytest-cov
     pytest-datadir
     ipython
+    pymech
 
 hpc =
     %(tests)s

diff --git a/src/snek5000/util/archive.py b/src/snek5000/util/archive.py
@@ -1,5 +1,4 @@
 """Post simulation archive-creation utilities"""
-import re
 import shlex
 import shutil
 import subprocess
@@ -9,6 +8,7 @@
 
 from .. import logger
 from . import modification_date
+from .files import next_path
 
 
 def archive(tarball, items=(), remove=False, readonly=False):
@@ -113,33 +113,6 @@ def exec_tar(tarball, items, remove):
     return cmd_output, output
 
 
-def next_path(old_path):
-    """Generate a new path with an integer suffix
-
-    Example
-    -------
-    >>> Path("test.txt").touch()
-    >>> next_path("test.txt")
-    test_1.txt # if path exists
-    test.txt # if path does not exists
-
-    """
-    i = 1
-    p = Path(old_path)
-    while p.exists():
-        # for example: remove .tar from the end, if any
-        stem = p.stem
-        for suffix in p.suffixes:
-            stem = re.sub(f"{suffix}$", "", stem)
-
-        p = p.parent / "".join([stem, f"_{i:02d}", *p.suffixes])
-        logger.info(f"Checking if path exists: {p}")
-
-    logger.info(f"Output path: {p}")
-
-    return p
-
-
 def parse_args_from_filename(tarball):
     tarball = Path(str(tarball))
 

diff --git a/src/snek5000/util/files.py b/src/snek5000/util/files.py
@@ -0,0 +1,72 @@
+import re
+from pathlib import Path
+
+from .. import logger
+
+
+def next_path(old_path, force_suffix=False):
+    """Generate a new path with an integer suffix
+
+    Parameters
+    ----------
+    old_path: str or path-like
+        Path to check for existence
+
+    force_suffix:
+        If true, will not check if the `old_path` can be used and adds
+        a suffix in the end.
+
+    Returns
+    -------
+    new_path: Path
+        A path (with an integer suffix) which does not yet exist in the
+        filesystem.
+
+    Example
+    -------
+    >>> import os
+    >>> os.chdir("/tmp")
+
+    >>> next_path("test.txt")  # path does not exist
+    PosixPath('test.txt')
+
+    >>> next_path("test.txt", force_suffix=True)  # path does not exists
+    PosixPath('test_00.txt')
+
+    >>> Path("test.txt").touch()
+    >>> next_path("test.txt")  # path exists
+    PosixPath('test_00.txt')
+
+    >>> Path("test_00.txt").touch()
+    >>> next_path("test.txt")  # path and the next one both exists
+    PosixPath('test_01.txt')
+
+    >>> Path("test.txt").unlink()  # cleanup
+    >>> Path("test_00.txt").unlink()
+
+    """
+
+    def int_suffix(p, integer):
+        stem = p.stem
+        # for example: remove .tar from the end, if any
+        for suffix in p.suffixes:
+            stem = re.sub(f"{suffix}$", "", stem)
+
+        return p.parent / "".join([stem, f"_{integer:02d}", *p.suffixes])
+
+    old_path = Path(old_path)
+
+    if not force_suffix and not old_path.exists():
+        return old_path
+
+    i = 0
+    new_path = int_suffix(old_path, i)
+
+    while new_path.exists():
+        logger.debug(f"Checking if path exists: {new_path}")
+        new_path = int_suffix(old_path, i)
+        i += 1
+
+    logger.debug(f"Next path available: {new_path}")
+
+    return new_path
diff --git a/src/snek5000/util/restart.py b/src/snek5000/util/restart.py
@@ -72,7 +72,7 @@ def get_status(path, verbose=False):
 
     Parameters
     ----------
-    path : str
+    path : str or path-like
         Path to an existing simulation directory
     verbose : bool
         Print out the path and its contents
@@ -83,6 +83,7 @@ def get_status(path, verbose=False):
         Enumeration indicating status code and message
 
     """
+    path = Path(path)
     locks_dir = path / ".snakemake" / "locks"
     contents = os.listdir(path)
 

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -99,6 +99,26 @@ def sim2d():
     return Simul(params)
 
 
+@pytest.fixture(scope="module")
+def sim_executed():
+    from phill.solver import Simul
+
+    params = Simul.create_default_params()
+    params.output.sub_directory = "test"
+
+    params.nek.general.stop_at = "endTime"
+    params.nek.general.end_time = 10 * abs(params.nek.general.dt)
+    params.nek.general.write_interval = 5
+
+    params.oper.nproc_min = 2
+    params.oper.nproc_max = 12
+    params.oper.nx = params.oper.ny = params.oper.nz = 3
+
+    sim = Simul(params)
+    sim.make.exec(["run_fg"])
+    return sim
+
+
 @pytest.fixture(scope="function")
 def sim_data(tmpdir_factory):
     """Generate fake simulation data."""

diff --git a/tests/test_files.py b/tests/test_files.py
@@ -0,0 +1,70 @@
+from pathlib import Path
+
+from snek5000.util import files
+
+
+def test_next_path_no_files(tmpdir):
+    tmpdir = Path(tmpdir)
+
+    # Create no files, multiple suffixes
+    target = tmpdir / "test.tar.gz"
+
+    assert str(files.next_path(target, force_suffix=True)) == str(
+        tmpdir / "test_00.tar.gz"
+    )
+
+
+def test_next_path_one_file(tmpdir):
+    tmpdir = Path(tmpdir)
+
+    # Create a file
+    target = tmpdir / "test.txt"
+    target.touch()  # Not necessary, but to see what happens.
+
+    assert str(files.next_path(target, force_suffix=True)) == str(
+        tmpdir / "test_00.txt"
+    )
+
+
+def test_next_path_one_file_no_suffix(tmpdir):
+    tmpdir = Path(tmpdir)
+
+    # Create a file
+    target = tmpdir / "test.txt"
+    target.touch()
+    (tmpdir / "test_00.txt").touch()
+
+    assert str(files.next_path(target)) == str(tmpdir / "test_01.txt")
+
+
+def test_next_path_two_files(tmpdir):
+    tmpdir = Path(tmpdir)
+
+    # Create multple files
+    target = tmpdir / "test.txt"
+    (tmpdir / "test_00.txt").touch()
+    (tmpdir / "test_01.txt").touch()
+
+    assert str(files.next_path(target, force_suffix=True)) == str(
+        tmpdir / "test_02.txt"
+    )
+
+
+def test_next_path_dir_no_suffix(tmpdir):
+    tmpdir = Path(tmpdir)
+
+    # Create a directory
+    target = tmpdir / "test_dir"
+    target.mkdir()
+
+    assert str(files.next_path(target)) == str(tmpdir / "test_dir_00")
+
+
+def test_next_path_dir(tmpdir):
+    tmpdir = Path(tmpdir)
+
+    target = tmpdir / "test_dir"
+
+    assert str(files.next_path(target, force_suffix=True)) == str(
+        tmpdir / "test_dir_00"
+    )
diff --git a/tests/test_restart.py b/tests/test_restart.py
@@ -1,3 +1,4 @@
+import pymech as pm
 import pytest
 
 from snek5000.util.restart import SnekRestartError, get_status, load_for_restart
@@ -45,5 +46,28 @@ def test_partial_content(sim_data):
     assert get_status(sim_data).code == 206
 
 
-def test_restart(sim_data):
-    pass
+def test_restart_error(tmpdir):
+    with pytest.raises(SnekRestartError, match="425: Too Early"):
+        load_for_restart(tmpdir)
+
+
+@pytest.mark.slow
+def test_restart(sim_executed):
+    params, Simul = load_for_restart(sim_executed.path_run)
+
+    # TODO: overwrite params xml and par file
+    sim = Simul(params)
+    case = sim.info_solver.short_name
+
+    # TODO: easier mechanism to load the last file
+    fld = pm.readnek(sorted(sim.path_run.glob(f"{case}0.f?????"))[-1])
+    t_end = params.nek.general.end_time = fld.time + 10 * abs(
+        params.nek.general.dt
+    )  # In phill for some reason dt is negative
+
+    params.nek._write_par(sim.path_run / f"{case}.par")
+
+    sim.make.exec(["run_fg"])
+
+    fld = pm.readnek(sorted(sim.path_run.glob(f"{case}0.f?????"))[-1])
+    assert fld.time == t_end
-Original file line number
+Diff line change
@@ Expand Up / @@ -55,6 +55,7 @@ tests = @@
         pytest-cov
         pytest-datadir
         ipython
+        pymech
     hpc =
         %(tests)s
@@ Expand Down @@