general: clean up project structure and sync ci configs with pymplate

.ci/release

@@ -29,7 +29,7 @@ def main() -> None:
 
     extra = []
     if args.test:
-        extra.extend(['--repository-url', 'https://test.pypi.org/legacy/'])
+        extra.extend(['--repository', 'testpypi'])
 
     root = Path(__file__).absolute().parent.parent
     os.chdir(root) # just in case
@@ -42,7 +42,7 @@ def main() -> None:
     if dist.exists():
         shutil.rmtree(dist)
 
-    check_call('python3 setup.py sdist bdist_wheel', shell=True)
+    check_call(['python3', '-m', 'build'])
 
     TP = 'TWINE_PASSWORD'
     password = os.environ.get(TP)

.ci/run

@@ -14,16 +14,17 @@ fi
 if [ -n "${CI-}" ]; then
     # install OS specific stuff here
     case "$OSTYPE" in
-    cygwin* | msys* | win*)
-        # windows
-        :
-        ;;
     darwin*) 
         # macos
         command -v vim    || brew install vim
         command -v pandoc || brew install pandoc
         ;;
+    cygwin* | msys* | win*)
+        # windows
+        :
+        ;;
     *)
+        # must be linux?
         # vim is used in one of the tests
         # we only run it on linux for simplicity
         command -v vim    || sudo apt install vim
@@ -40,4 +41,4 @@ if ! command -v python3 &> /dev/null; then
 fi
 
 "$PY_BIN" -m pip install --user tox
-"$PY_BIN" -m tox
+"$PY_BIN" -m tox --parallel --parallel-live "$@"

.github/workflows/main.yml

@@ -10,15 +10,22 @@ on:
   pull_request: # needed to trigger on others' PRs
   # Note that people who fork it need to go to "Actions" tab on their fork and click "I understand my workflows, go ahead and enable them".
   workflow_dispatch: # needed to trigger workflows manually
-  # todo cron?
+    # todo cron?
+    inputs:
+      debug_enabled:
+        type: boolean
+        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
+        required: false
+        default: false
 
 
 jobs:
   build:
     strategy:
+      fail-fast: false
       matrix:
-        platform: [ubuntu-latest, macos-latest] # todo enable windows later -- need to fix a couple of tests first, windows-latest]
-        python-version: ['3.7', '3.8', '3.9', '3.10']
+        platform: [ubuntu-latest, macos-latest]  # todo enable windows-latest later -- need to fix a couple of tests first
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
         # vvv just an example of excluding stuff from matrix
         # exclude: [{platform: macos-latest, python-version: '3.6'}]
 
@@ -28,22 +35,23 @@ jobs:
     # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation
     - run: echo "$HOME/.local/bin" >> $GITHUB_PATH
 
-    - uses: actions/setup-python@v3
+    - uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 
     - uses: actions/checkout@v3
       with:
         submodules: recursive
+        fetch-depth: 0  # nicer to have all git history when debugging/for tests
 
-    # uncomment for SSH debugging
-    # - uses: mxschmitt/action-tmate@v3
+    - uses: mxschmitt/action-tmate@v3
+      if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
 
     # explicit bash command is necessary for Windows CI runner, otherwise it thinks it's cmd...
     - run: bash .ci/run
 
     - if: matrix.platform == 'ubuntu-latest'  # no need to compute coverage for other platforms
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
         name: .coverage.mypy_${{ matrix.platform }}_${{ matrix.python-version }}
         path: .coverage.mypy/
@@ -57,7 +65,7 @@ jobs:
     # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation
     - run: echo "$HOME/.local/bin" >> $GITHUB_PATH
 
-    - uses: actions/setup-python@v3
+    - uses: actions/setup-python@v4
       with:
         python-version: '3.8'
 
@@ -70,12 +78,12 @@ jobs:
       if: github.event_name != 'pull_request' && github.event.ref == 'refs/heads/master'
       env:
         TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD_TEST }}
-      run: pip3 install --user wheel twine && .ci/release --test
+      run: pip3 install --user --upgrade build twine && .ci/release --test
 
     - name: 'release to pypi'
       # always deploy tags to release pypi
       # NOTE: release tags are guarded by on: push: tags on the top
       if: github.event_name != 'pull_request' && startsWith(github.event.ref, 'refs/tags')
       env:
         TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
-      run: pip3 install --user wheel twine && .ci/release
+      run: pip3 install --user --upgrade build twine && .ci/release

conftest.py

@@ -0,0 +1,38 @@
+# this is a hack to monkey patch pytest so it handles tests inside namespace packages without __init__.py properly
+# without it, pytest can't discover the package root for some reason
+# also see https://github.com/karlicoss/pytest_namespace_pkgs for more
+
+import pathlib
+from typing import Optional
+
+import _pytest.main
+import _pytest.pathlib
+
+# we consider all dirs in repo/ to be namespace packages
+root_dir = pathlib.Path(__file__).absolute().parent.resolve() / 'src'
+assert root_dir.exists(), root_dir
+
+# TODO assert it contains package name?? maybe get it via setuptools..
+
+namespace_pkg_dirs = [str(d) for d in root_dir.iterdir() if d.is_dir()]
+
+# resolve_package_path is called from _pytest.pathlib.import_path
+# takes a full abs path to the test file and needs to return the path to the 'root' package on the filesystem
+resolve_pkg_path_orig = _pytest.pathlib.resolve_package_path
+def resolve_package_path(path: pathlib.Path) -> Optional[pathlib.Path]:
+    result = path  # search from the test file upwards
+    for parent in result.parents:
+        if str(parent) in namespace_pkg_dirs:
+            return parent
+    raise RuntimeError("Couldn't determine path for ", path)
+_pytest.pathlib.resolve_package_path = resolve_package_path
+
+
+# without patching, the orig function returns just a package name for some reason
+# (I think it's used as a sort of fallback)
+# so we need to point it at the absolute path properly
+# not sure what are the consequences.. maybe it wouldn't be able to run against installed packages? not sure..
+search_pypath_orig = _pytest.main.search_pypath
+def search_pypath(module_name: str) -> str:
+    return str(root_dir)
+_pytest.main.search_pypath = search_pypath

mypy.ini

@@ -1,10 +1,15 @@
 [mypy]
+namespace_packages = True
 pretty = True
 show_error_context = True
 show_error_codes = True
+show_column_numbers = True
+show_error_end = True
+warn_unused_ignores = True
 check_untyped_defs = True
-namespace_packages = True
+enable_error_code = possibly-undefined
+strict_equality = True
 
-# it's not controled by me, so for now just ignore..
-[mypy-my.config.repos.pdfannots.pdfannots]
-ignore_errors = True
+# an example of suppressing
+# [mypy-my.config.repos.pdfannots.pdfannots]
+# ignore_errors = True

pyproject.toml

@@ -0,0 +1,40 @@
+# see https://github.com/karlicoss/pymplate for up-to-date reference
+[project]
+dynamic = ["version"]  # version is managed by setuptools_scm
+name = "orger"
+dependencies = [
+    "appdirs"     , # to keep state files
+    "atomicwrites", # to safely append data to a file
+]
+
+## these need to be set if you're planning to upload to pypi
+description = "Converts data into org-mode"
+license = {file = "LICENSE"}
+authors = [
+    {name = "Dima Gerasimov (@karlicoss)", email = "[email protected]"},
+]
+maintainers = [
+    {name = "Dima Gerasimov (@karlicoss)", email = "[email protected]"},
+]
+
+
+[project.urls]
+Homepage = "https://github.com/karlicoss/orger"
+##
+
+[project.optional-dependencies]
+testing = [
+    "pytest",
+    "ruff",
+    "mypy",
+    "lxml",  # for mypy html coverage
+]
+
+
+[build-system]
+requires = ["setuptools", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
+version_scheme = "python-simplified-semver"
+local_scheme = "dirty-tag"

pytest.ini

@@ -11,5 +11,3 @@ addopts =
 
   # show all test durations (unless they are too short)
   --durations=0
-
-  --ignore=src/orger/modules

ruff.toml

@@ -0,0 +1,25 @@
+ignore = [
+### too opinionated style checks
+    "E501",  # too long lines
+    "E702",  # Multiple statements on one line (semicolon)
+    "E731",  # assigning lambda instead of using def
+    "E741",  # Ambiguous variable name: `l`
+    "E742",  # Ambiguous class name: `O
+    "E401",  # Multiple imports on one line
+    "F403",  # import *` used; unable to detect undefined names
+###
+
+###
+    "E722",  # Do not use bare `except` ## Sometimes it's useful for defensive imports and that sort of thing..
+    "F811",  # Redefinition of unused  # this gets in the way of pytest fixtures (e.g. in cachew)
+
+## might be nice .. but later and I don't wanna make it strict
+    "E402",  # Module level import not at top of file
+
+### maybe consider these soon
+# sometimes it's useful to give a variable a name even if we don't use it as a documentation
+# on the other hand, often is a sign of error
+    "F841",  # Local variable `count` is assigned to but never used
+    "F401",  # imported but unused
+###
+]

src/orger/atomic_append.py

@@ -45,7 +45,7 @@ def atomic_append_check(
 
 
 def test_atomic_append_check(tmp_path: Path) -> None:
-    import pytest # type: ignore
+    import pytest
     import platform
 
     if platform.system() == 'Windows':

src/orger/common.py

@@ -33,7 +33,7 @@ def dt_heading(dt: Optional[datetime], heading: str) -> str:
 def error(e: Exception) -> OrgNode:
     import traceback
     return OrgNode(
-        heading=f"ERROR!",
+        heading="ERROR!",
         body='\n'.join(traceback.format_exception(Exception, e, None)),
     )
 
@@ -58,5 +58,5 @@ def todo(dt: datetime, **kwargs):
 
 
 def orger_user_dir() -> Path:
-    import appdirs # type: ignore[import]
+    import appdirs
     return Path(appdirs.user_config_dir('orger'))

src/orger/klogging.py

@@ -51,7 +51,7 @@ def isEnabledFor_lazyinit(*args, logger=logger, orig=logger.isEnabledFor, **kwar
                 setattr(logger, att, True)
             return orig(*args, **kwargs)
 
-        logger.isEnabledFor = isEnabledFor_lazyinit  # type: ignore[assignment]
+        logger.isEnabledFor = isEnabledFor_lazyinit  # type: ignore[method-assign]
         return cast(LazyLogger, logger)
 
 

modules/tinder.py → src/orger/modules/tinder.py

@@ -6,7 +6,7 @@
 
 import my.config
 # todo later, use .all?
-import my.tinder.android as tinder  # type: ignore[import] # uncomment once we release newer hpi
+import my.tinder.android as tinder
 
 class TinderView(Mirror):
     def get_items(self) -> Mirror.Results:

src/orger/org_view.py

@@ -22,7 +22,7 @@
 
 _style_map: Dict[str, TimestampStyle] = {
     k.lower(): v  # type: ignore[misc]
-    for k, v in TimestampStyle._member_map_.items() # type: ignore[attr-defined]
+    for k, v in TimestampStyle._member_map_.items()
 }
 
 class OrgView:
@@ -67,7 +67,7 @@ def main_common(self) -> None:
     @classmethod
     def parser(cls) -> ArgumentParser:
         F = lambda prog: argparse.ArgumentDefaultsHelpFormatter(prog, width=120)
-        p = argparse.ArgumentParser(formatter_class=F) # type: ignore
+        p = argparse.ArgumentParser(formatter_class=F)
 
         p.add_argument(
             '--disable-pandoc',