From 3e3077abe143119e47681aa3f8812e5a3f2bc38d Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 7 Apr 2026 14:29:14 +0200 Subject: [PATCH 01/37] merged changes from issue 402 and changes-file --- doc/changes/unreleased.md | 1 + exasol/toolbox/nox/_dependencies.py | 22 ++- exasol/toolbox/util/dependencies/audit.py | 4 +- .../dependencies/track_vulnerabilities.py | 91 ++++++++---- exasol/toolbox/util/release/changes_file.py | 131 ++++++++++++++++++ test/unit/config_test.py | 16 ++- test/unit/nox/_dependencies_test.py | 33 +++-- .../track_vulnerabilities_test.py | 71 +++++----- test/unit/util/release/test_changes_file.py | 120 ++++++++++++++++ test/unit/util/release/test_section.py | 84 +++++++++++ 10 files changed, 495 insertions(+), 78 deletions(-) create mode 100644 exasol/toolbox/util/release/changes_file.py create mode 100644 test/unit/util/release/test_changes_file.py create mode 100644 test/unit/util/release/test_section.py diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 1a6832109..920f82f7a 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -15,6 +15,7 @@ To ensure usage of secure packages, it is up to the user to similarly relock the ## Features * #740: Added nox session `release:update` +* #402: Created nox task to detect resolved GitHub security issues ## Security Issues diff --git a/exasol/toolbox/nox/_dependencies.py b/exasol/toolbox/nox/_dependencies.py index dc860875a..d196dfd08 100644 --- a/exasol/toolbox/nox/_dependencies.py +++ b/exasol/toolbox/nox/_dependencies.py @@ -9,17 +9,21 @@ from exasol.toolbox.util.dependencies.audit import ( PipAuditException, Vulnerabilities, + get_vulnerabilities, + get_vulnerabilities_from_latest_tag, ) from exasol.toolbox.util.dependencies.licenses import ( PackageLicenseReport, get_licenses, ) from exasol.toolbox.util.dependencies.poetry_dependencies import get_dependencies +from exasol.toolbox.util.dependencies.track_vulnerabilities import DependenciesAudit +from noxconfig import PROJECT_CONFIG @nox.session(name="dependency:licenses", python=False) def dependency_licenses(session: Session) -> None: - """Return the packages with their licenses""" + """Report licenses for all dependencies.""" dependencies = get_dependencies(working_directory=Path()) licenses = get_licenses() license_markdown = PackageLicenseReport( @@ -28,9 +32,10 @@ def dependency_licenses(session: Session) -> None: print(license_markdown.to_markdown()) -@nox.session(name="dependency:audit", python=False) +# Probably this session is obsolete +@nox.session(name="dependency:audit-old", python=False) def audit(session: Session) -> None: - """Check for known vulnerabilities""" + """Report known vulnerabilities.""" try: vulnerabilities = Vulnerabilities.load_from_pip_audit(working_directory=Path()) @@ -39,3 +44,14 @@ def audit(session: Session) -> None: security_issue_dict = vulnerabilities.security_issue_dict print(json.dumps(security_issue_dict, indent=2)) + + +@nox.session(name="vulnerabilities:resolved", python=False) +def report_resolved_vulnerabilities(session: Session) -> None: + """Report resolved vulnerabilities in dependencies.""" + path = PROJECT_CONFIG.root_path + audit = DependenciesAudit( + previous_vulnerabilities=get_vulnerabilities_from_latest_tag(path), + current_vulnerabilities=get_vulnerabilities(path), + ) + print(audit.report_resolved_vulnerabilities()) diff --git a/exasol/toolbox/util/dependencies/audit.py b/exasol/toolbox/util/dependencies/audit.py index 53fb67352..836bf8382 100644 --- a/exasol/toolbox/util/dependencies/audit.py +++ b/exasol/toolbox/util/dependencies/audit.py @@ -177,7 +177,7 @@ def audit_poetry_files(working_directory: Path) -> str: tmpdir = Path(path) (tmpdir / requirements_txt).write_text(output.stdout) - command = ["pip-audit", "-r", requirements_txt, "-f", "json"] + command = ["pip-audit", "--disable-pip", "-r", requirements_txt, "-f", "json"] output = subprocess.run( command, capture_output=True, @@ -239,6 +239,6 @@ def get_vulnerabilities(working_directory: Path) -> list[Vulnerability]: ).vulnerabilities -def get_vulnerabilities_from_latest_tag(root_path: Path): +def get_vulnerabilities_from_latest_tag(root_path: Path) -> list[Vulnerability]: with poetry_files_from_latest_tag(root_path=root_path) as tmp_dir: return get_vulnerabilities(working_directory=tmp_dir) diff --git a/exasol/toolbox/util/dependencies/track_vulnerabilities.py b/exasol/toolbox/util/dependencies/track_vulnerabilities.py index d9d663797..ec079999c 100644 --- a/exasol/toolbox/util/dependencies/track_vulnerabilities.py +++ b/exasol/toolbox/util/dependencies/track_vulnerabilities.py @@ -1,3 +1,6 @@ +from inspect import cleandoc +from typing import Dict + from pydantic import ( BaseModel, ConfigDict, @@ -6,37 +9,75 @@ from exasol.toolbox.util.dependencies.audit import Vulnerability -class ResolvedVulnerabilities(BaseModel): - model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) - - previous_vulnerabilities: list[Vulnerability] - current_vulnerabilities: list[Vulnerability] +class VulnerabilityMatcher: + def __init__(self, current_vulnerabilities: list[Vulnerability]): + # Dict of current vulnerabilities: + # * keys: package names + # * values: set of each vulnerability's references + self._references = { + v.package.name: set(v.references) + for v in current_vulnerabilities + } - def _is_resolved(self, previous_vuln: Vulnerability): + def is_resolved(self, vuln: Vulnerability) -> bool: """ Detects if a vulnerability has been resolved. - A vulnerability is said to be resolved when it cannot be found - in the `current_vulnerabilities`. In order to see if a vulnerability - is still present, its id and aliases are compared to values in the - `current_vulnerabilities`. It is hoped that if an ID were to change - that this would still be present in the aliases. + A vulnerability is said to be resolved when it cannot be found in + the `current_vulnerabilities`. + + Vulnerabilities are matched by the name of the affected package + and the vulnerability's "references" (set of ID and aliases). + + The vulnerability is rated as "resolved" only if there is not + intersection between previous and current references. + + This hopefully compensates in case a different ID is assigned to a + vulnerability. """ - previous_vuln_set = {previous_vuln.id, *previous_vuln.aliases} - for current_vuln in self.current_vulnerabilities: - if previous_vuln.package.name == current_vuln.package.name: - current_vuln_id_set = {current_vuln.id, *current_vuln.aliases} - if previous_vuln_set.intersection(current_vuln_id_set): - return False - return True + refs = set(vuln.references) + current = self._references.get(vuln.package.name, set()) + return not refs.intersection(current) + + +class DependenciesAudit(BaseModel): + """ + Compare previous vulnerabilities to current ones and create a report + about the resolved vulnerabilities. + """ + + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + previous_vulnerabilities: list[Vulnerability] + current_vulnerabilities: list[Vulnerability] @property - def resolutions(self) -> list[Vulnerability]: + def resolved_vulnerabilities(self) -> list[Vulnerability]: """ - Return resolved vulnerabilities + Return the list of resolved vulnerabilities. """ - resolved_vulnerabilities = [] - for previous_vuln in self.previous_vulnerabilities: - if self._is_resolved(previous_vuln): - resolved_vulnerabilities.append(previous_vuln) - return resolved_vulnerabilities + matcher = VulnerabilityMatcher(self.current_vulnerabilities) + return [ + vuln for vuln in self.previous_vulnerabilities + if matcher.is_resolved(vuln) + ] + + def report_resolved_vulnerabilities(self) -> str: + if not (resolved := self.resolved_vulnerabilities): + return "" + header = cleandoc( + """ + ## Fixed Vulnerabilities + + This release fixes vulnerabilities by updating dependencies: + + | Dependency | Vulnerability | Affected | Fixed in | + |------------|---------------|----------|----------| + """ + ) + def formatted(vuln: Vulnerability) -> str: + columns = (vuln.package.name, vuln.id, str(vuln.package.version), vuln.fix_versions[0]) + return f'| {" | ".join(columns)} |' + + body = "\n".join(formatted(v) for v in resolved) + return f"{header}\n{body}" diff --git a/exasol/toolbox/util/release/changes_file.py b/exasol/toolbox/util/release/changes_file.py new file mode 100644 index 000000000..f1f702142 --- /dev/null +++ b/exasol/toolbox/util/release/changes_file.py @@ -0,0 +1,131 @@ +""" +A project's Changelog is expected to list the changes coming with each of +the project's releases. The Changelog contains a file changelog.md with the +table of contents. and zero or more changes files. All files are in Markdown +syntax. + +Each changes file is named changes_*.md and describes the changes for a +specific release. The * in the file name is identical to the version number of +the related release. + +Each changes file starts with a section describing the version number, date +and name of the release and one or more subsections. The first subsection is a +summary, each other subsection lists the issues (aka. tickets) of a particular +category the are resolved by this release. Categories are security, bugfixes, +features, documentation, and refactorings. + +For the sake of simplicity, class ChangesFile maintains the sections as a +sequence, ignoring their hierarchy. + +Each section is identified by its title which should be unique. + +A section may consist of a prefix and a suffix, either might be empty. The +prefix are some introductory sentences, the suffix is the list of issues in +this section. + +Method Section.replace_prefix() adds such a prefix or replaces it, when the +section already has one. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from inspect import cleandoc + + +class ParseError(Exception): + """ + Indicates inconsistencies when parsing a changelog from raw + text. E.g. a section with a body but no title. + """ + + +@dataclass +class Section: + title: str + body: str + + @property + def rendered(self) -> str: + return f"{self.title}\n\n{self.body}" + + def replace_prefix(self, prefix: str) -> None: + """ + Prepends the specified prefix to the body of the section. + + If the body starts with the first line of the specified prefix, then + replace the body's prefix. + """ + flags = re.DOTALL | re.MULTILINE + if not self.body.startswith(prefix.splitlines()[0]): + self.body = f"{prefix}\n\n{self.body}" if self.body else prefix + elif re.search(r"^[*-] ", self.body, flags=flags): + suffix = re.sub(r".*?^([*-])", r"\1", self.body, count=1, flags=flags) + self.body = f"{prefix}\n\n{suffix}" + else: + self.body = prefix + + +@dataclass +class ChangesFile: + """ + Represents file unreleased.md or changes_*.py in folder doc/changes/. + """ + + sections: list[Section] + + def get_section(self, title: str) -> Section | None: + """ + Retrieve the section with the specified title. + """ + + pattern = re.compile(f"#+ {re.escape(title)}$") + return next((s for s in self.sections if pattern.match(s.title)), None) + + def add_section(self, section: Section, pos: int = 1) -> None: + """ + Insert the specified section at the specified position. + """ + + self.sections.insert(pos, section) + + @property + def rendered(self) -> str: + return "\n\n".join(s.rendered for s in self.sections) + + @classmethod + def parse(cls, content: str) -> ChangesFile: + title = None + body = [] + sections = [] + + def is_body(line: str) -> bool: + return not line.startswith("#") + + def process_section(): + nonlocal sections + if title: + sections.append(Section(title, cleandoc("\n".join(body)))) + + for line in content.splitlines(): + if is_body(line): + if not title: + raise ParseError(f"Found body line without preceding title: {line}") + body.append(line) + continue + # found new title + process_section() + title = line + body = [] + + process_section() + return ChangesFile(sections) + + +def sample(): + changes = ChangesFile.parse(content) + if section := changes.get_section(title): + section.replace_prefix(body) + else: + changes.add_section(Section(title, body)) diff --git a/test/unit/config_test.py b/test/unit/config_test.py index 385ab2aad..035f7a6b0 100644 --- a/test/unit/config_test.py +++ b/test/unit/config_test.py @@ -1,5 +1,6 @@ from collections.abc import Iterable from pathlib import Path +from unittest.mock import Mock import pytest from pydantic_core._pydantic_core import ValidationError @@ -9,6 +10,7 @@ BaseConfig, DependencyManager, valid_version_string, + warnings, ) from exasol.toolbox.nox.plugin import hookimpl from exasol.toolbox.util.version import Version @@ -202,9 +204,19 @@ def test_raises_exception_without_hook(test_project_config_factory): class TestDependencyManager: @staticmethod - @pytest.mark.parametrize("version", ["2.1.4", "2.3.0", "2.9.9"]) - def test_works_as_expected(version): + @pytest.mark.parametrize( + "version, expected_warning", + [ + ("2.1.4", None), + ("2.3.0", None), + ("2.9.9", "Poetry version exceeds last tested version"), + ], + ) + def test_works_as_expected(version, expected_warning, monkeypatch): + monkeypatch.setattr(warnings, "warn", Mock()) DependencyManager(name="poetry", version=version) + if expected_warning: + assert expected_warning in warnings.warn.call_args.args[0] @staticmethod def test_raises_exception_when_not_supported_name(): diff --git a/test/unit/nox/_dependencies_test.py b/test/unit/nox/_dependencies_test.py index b841d46d3..a391193e4 100644 --- a/test/unit/nox/_dependencies_test.py +++ b/test/unit/nox/_dependencies_test.py @@ -1,18 +1,25 @@ -from unittest import mock +from unittest.mock import Mock -from exasol.toolbox.nox._dependencies import audit +from exasol.toolbox.nox import _dependencies from exasol.toolbox.util.dependencies.audit import Vulnerabilities -class TestAudit: - @staticmethod - def test_works_as_expected_with_mock(nox_session, sample_vulnerability, capsys): - with mock.patch( - "exasol.toolbox.nox._dependencies.Vulnerabilities" - ) as mock_class: - mock_class.load_from_pip_audit.return_value = Vulnerabilities( - vulnerabilities=[sample_vulnerability.vulnerability] - ) - audit(nox_session) +# Proposal: Remove this test and the related nox task under test +def test_audit(monkeypatch, nox_session, sample_vulnerability, capsys): + monkeypatch.setattr(_dependencies, "Vulnerabilities", Mock()) + _dependencies.Vulnerabilities.load_from_pip_audit.return_value = Vulnerabilities( + vulnerabilities=[sample_vulnerability.vulnerability] + ) + _dependencies.audit(nox_session) + assert capsys.readouterr().out == sample_vulnerability.nox_dependencies_audit - assert capsys.readouterr().out == sample_vulnerability.nox_dependencies_audit + +def test_report_resolved_vulnerabilities(monkeypatch, nox_session, capsys, sample_vulnerability): + monkeypatch.setattr( + _dependencies, + "get_vulnerabilities_from_latest_tag", + Mock(return_value=[sample_vulnerability.vulnerability]), + ) + monkeypatch.setattr(_dependencies, "get_vulnerabilities", Mock(return_value=[])) + _dependencies.report_resolved_vulnerabilities(nox_session) + assert "| jinja2 | CVE-2025-27516 | 3.1.5 | 3.1.6 |" in capsys.readouterr().out diff --git a/test/unit/util/dependencies/track_vulnerabilities_test.py b/test/unit/util/dependencies/track_vulnerabilities_test.py index 1dd584ac9..cdfae905b 100644 --- a/test/unit/util/dependencies/track_vulnerabilities_test.py +++ b/test/unit/util/dependencies/track_vulnerabilities_test.py @@ -1,55 +1,60 @@ +from exasol.toolbox.util.dependencies.audit import Vulnerability from exasol.toolbox.util.dependencies.track_vulnerabilities import ( - ResolvedVulnerabilities, + DependenciesAudit, + VulnerabilityMatcher, ) -class TestResolvedVulnerabilities: - def test_vulnerability_present_for_previous_and_current(self, sample_vulnerability): +def _flip_id_and_alias(vulnerability: SampleVulnerability): + other = vulnerability + vuln_entry = { + "aliases": [other.vulnerability_id], + "id": other.cve_id, + "fix_versions": other.vulnerability.fix_versions, + "description": other.description, + } + return Vulnerability.from_audit_entry( + package_name=other.package_name, + version=other.version, + vuln_entry=vuln_entry, + ) + + +class TestVulnerabilityMatcher: + def test_not_resolved(self, sample_vulnerability): vuln = sample_vulnerability.vulnerability - resolved = ResolvedVulnerabilities( - previous_vulnerabilities=[vuln], current_vulnerabilities=[vuln] - ) - assert resolved._is_resolved(vuln) is False - - def test_vulnerability_present_for_previous_and_current_with_different_id( - self, sample_vulnerability - ): - vuln2 = sample_vulnerability.vulnerability.__dict__.copy() - vuln2["version"] = sample_vulnerability.version - # flipping aliases & id to ensure can match across types - vuln2["aliases"] = [sample_vulnerability.vulnerability_id] - vuln2["id"] = sample_vulnerability.cve_id - - resolved = ResolvedVulnerabilities( - previous_vulnerabilities=[sample_vulnerability.vulnerability], - current_vulnerabilities=[vuln2], - ) - assert resolved._is_resolved(sample_vulnerability.vulnerability) is False + matcher = VulnerabilityMatcher(current_vulnerabilities=[vuln]) + assert not matcher.is_resolved(vuln) + + def test_changed_id_not_resolved(self, sample_vulnerability): + vuln2 = _flip_id_and_alias(sample_vulnerability) + matcher = VulnerabilityMatcher(current_vulnerabilities=[vuln2]) + assert not matcher.is_resolved(sample_vulnerability.vulnerability) - def test_vulnerability_in_previous_resolved_in_current(self, sample_vulnerability): + def test_resolved(self, sample_vulnerability): vuln = sample_vulnerability.vulnerability - resolved = ResolvedVulnerabilities( - previous_vulnerabilities=[vuln], current_vulnerabilities=[] - ) - assert resolved._is_resolved(vuln) is True + matcher = VulnerabilityMatcher(current_vulnerabilities=[]) + assert matcher.is_resolved(vuln) + +class TestDependenciesAudit: def test_no_vulnerabilities_for_previous_and_current(self): - resolved = ResolvedVulnerabilities( + audit = DependenciesAudit( previous_vulnerabilities=[], current_vulnerabilities=[] ) - assert resolved.resolutions == [] + assert audit.resolved_vulnerabilities == [] def test_vulnerability_in_current_but_not_present(self, sample_vulnerability): - resolved = ResolvedVulnerabilities( + audit = DependenciesAudit( previous_vulnerabilities=[], current_vulnerabilities=[sample_vulnerability.vulnerability], ) # only care about "resolved" vulnerabilities, not new ones - assert resolved.resolutions == [] + assert audit.resolved_vulnerabilities == [] def test_resolved_vulnerabilities(self, sample_vulnerability): - resolved = ResolvedVulnerabilities( + audit = DependenciesAudit( previous_vulnerabilities=[sample_vulnerability.vulnerability], current_vulnerabilities=[], ) - assert resolved.resolutions == [sample_vulnerability.vulnerability] + assert audit.resolved_vulnerabilities == [sample_vulnerability.vulnerability] diff --git a/test/unit/util/release/test_changes_file.py b/test/unit/util/release/test_changes_file.py new file mode 100644 index 000000000..2d5f80dd5 --- /dev/null +++ b/test/unit/util/release/test_changes_file.py @@ -0,0 +1,120 @@ +from inspect import cleandoc + +import pytest + +from exasol.toolbox.util.release.changes_file import ( + ChangesFile, + ParseError, + Section, +) + +import pytest + + +class Scenario: + def __init__(self, initial: str, expected_output: str, expected_sections: list[str]): + self.testee = ChangesFile.parse(cleandoc(initial)) + self.expected_output = cleandoc(expected_output) + self.expected_sections = expected_sections + + +EMPTY = Scenario( + initial="", + expected_output="", + expected_sections=[], +) + +MINIMAL = Scenario( + initial=""" + # title + body + """, + expected_output=""" + # title + + body + """, + expected_sections=["title"], +) + +SPECIAL_CHAR_TITLE = "+ special [char] * title" + +SPECIAL_CHAR_SECTION = Scenario( + initial=f""" + # {SPECIAL_CHAR_TITLE} + body + """, + expected_output=""" + # {SPECIAL_CHAR_TITLE} + + body + """, + expected_sections=[SPECIAL_CHAR_TITLE], +) + +WITH_SUBSECTION = Scenario( + initial=""" + # title + body + ## subtitle + paragraph + + * item 1 + * item 2 + """, + expected_output=""" + # title + + body + + ## subtitle + + paragraph + + * item 1 + * item 2 + """, + expected_sections=["title","subtitle"] + ) + + +def test_parse_error() -> None: + with pytest.raises(ParseError, match="Found body line without preceding title"): + ChangesFile.parse("body line") + + +@pytest.mark.parametrize("scenario", [EMPTY, MINIMAL, WITH_SUBSECTION]) +def test_number_of_sections(scenario: Scenario): + assert len(scenario.testee.sections) == len(scenario.expected_sections) + + +@pytest.mark.parametrize("scenario", [EMPTY, MINIMAL, SPECIAL_CHAR_SECTION, WITH_SUBSECTION]) +def test_get(scenario: Scenario): + assert all(scenario.testee.get_section(s) for s in scenario.expected_sections) + + +@pytest.mark.parametrize("scenario", [EMPTY, MINIMAL, WITH_SUBSECTION]) +def test_missing_section(scenario: Scenario): + assert scenario.testee.get_section("non existing") is None + + +@pytest.mark.parametrize("scenario", [EMPTY, MINIMAL, WITH_SUBSECTION]) +def test_render(scenario: Scenario): + assert scenario.testee.rendered == scenario.expected_output + + +@pytest.fixture +def sample_section(): + return Section("# blabla", "body") + + +@pytest.mark.parametrize("scenario", [MINIMAL, WITH_SUBSECTION]) +def test_add_non_empty(scenario: Scenario, sample_section): + scenario.testee.add_section(sample_section) + assert scenario.testee.sections[1] == sample_section + + +@pytest.mark.parametrize("scenario", [EMPTY]) +def test_add_empty(scenario: Scenario, sample_section): + scenario.testee.add_section(sample_section) + assert scenario.testee.sections[0] == sample_section diff --git a/test/unit/util/release/test_section.py b/test/unit/util/release/test_section.py new file mode 100644 index 000000000..b12e9d28f --- /dev/null +++ b/test/unit/util/release/test_section.py @@ -0,0 +1,84 @@ +from inspect import cleandoc + +import pytest + +from exasol.toolbox.util.release.changes_file import Section + + +class Scenario: + def __init__(self, body: str, expected_suffix: str): + self.body = cleandoc(body) + self.expected_suffix = ( + f"\n\n{cleandoc(expected_suffix)}" if expected_suffix else "" + ) + + def create_testee(self) -> Section: + return Section("# title", self.body) + + +NO_MATCHING_PREFIX = Scenario("body", expected_suffix="body") + +MATCHING_PREFIX_BUT_NO_LIST = Scenario( + body=""" + Prefix first line + + Another line + """, + expected_suffix="", +) + +MATCHING_PREFIX_AND_LIST = Scenario( + body=""" + Prefix first line + + Another line + + * item 1 + * item 2 + """, + expected_suffix=""" + * item 1 + * item 2 + """, +) + + +LIST_WITH_DASHES = Scenario( + body=""" + Prefix first line + + Another line + + - item 1 + - item 2 + """, + expected_suffix=""" + - item 1 + - item 2 + """, +) + + +SAMPLE_PREFIX = cleandoc(""" + Prefix first line + + | col 1 | col 2 | + |-------|-------| + | abc | 123 | + """) + + +@pytest.mark.parametrize( + "scenario", + [ + pytest.param(NO_MATCHING_PREFIX, id="no_matching_prefix"), + pytest.param(MATCHING_PREFIX_BUT_NO_LIST, id="matching_prefix_but_no_list"), + pytest.param(MATCHING_PREFIX_AND_LIST, id="matching_prefix_and_list"), + pytest.param(LIST_WITH_DASHES, id="list_with_dashes"), + ], +) +def test_replace_prefix(scenario): + testee = scenario.create_testee() + testee.replace_prefix(SAMPLE_PREFIX) + expected = f"{SAMPLE_PREFIX}{scenario.expected_suffix}" + assert testee.body == f"{SAMPLE_PREFIX}{scenario.expected_suffix}" From 753fa257a0e3d14f6b1cf90eb72685799a4edf6e Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 7 Apr 2026 15:02:54 +0200 Subject: [PATCH 02/37] Prepared using changes_file --- exasol/toolbox/util/release/changelog.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/exasol/toolbox/util/release/changelog.py b/exasol/toolbox/util/release/changelog.py index d82c0d2cc..35027536f 100644 --- a/exasol/toolbox/util/release/changelog.py +++ b/exasol/toolbox/util/release/changelog.py @@ -22,6 +22,9 @@ DEPENDENCY_UPDATES = "## Dependency Updates\n" +TITLE = "Dependency Updates" +HEADING = "## {TITLE}\n" + class Changelogs: def __init__(self, changes_path: Path, root_path: Path, version: Version) -> None: @@ -149,7 +152,7 @@ def get_changed_files(self) -> list[Path]: def _report_dependency_changes(self) -> str: if changes := self._dependency_changes(): - return f"{DEPENDENCY_UPDATES}{changes}" + return f"## {TITLE}\n{changes}" return "" def update_latest(self) -> Changelogs: From bbe64e1a1e0b7e0ab3fe41716b0b2cbb947307af Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 8 Apr 2026 10:27:38 +0200 Subject: [PATCH 03/37] merged latest changes from changes-file --- exasol/toolbox/util/release/changes_file.py | 131 ------------ exasol/toolbox/util/release/markdown.py | 156 ++++++++++++++ test/unit/util/release/test_changes_file.py | 120 ----------- test/unit/util/release/test_markdown.py | 216 ++++++++++++++++++++ test/unit/util/release/test_section.py | 84 -------- 5 files changed, 372 insertions(+), 335 deletions(-) delete mode 100644 exasol/toolbox/util/release/changes_file.py create mode 100644 exasol/toolbox/util/release/markdown.py delete mode 100644 test/unit/util/release/test_changes_file.py create mode 100644 test/unit/util/release/test_markdown.py delete mode 100644 test/unit/util/release/test_section.py diff --git a/exasol/toolbox/util/release/changes_file.py b/exasol/toolbox/util/release/changes_file.py deleted file mode 100644 index f1f702142..000000000 --- a/exasol/toolbox/util/release/changes_file.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -A project's Changelog is expected to list the changes coming with each of -the project's releases. The Changelog contains a file changelog.md with the -table of contents. and zero or more changes files. All files are in Markdown -syntax. - -Each changes file is named changes_*.md and describes the changes for a -specific release. The * in the file name is identical to the version number of -the related release. - -Each changes file starts with a section describing the version number, date -and name of the release and one or more subsections. The first subsection is a -summary, each other subsection lists the issues (aka. tickets) of a particular -category the are resolved by this release. Categories are security, bugfixes, -features, documentation, and refactorings. - -For the sake of simplicity, class ChangesFile maintains the sections as a -sequence, ignoring their hierarchy. - -Each section is identified by its title which should be unique. - -A section may consist of a prefix and a suffix, either might be empty. The -prefix are some introductory sentences, the suffix is the list of issues in -this section. - -Method Section.replace_prefix() adds such a prefix or replaces it, when the -section already has one. -""" - -from __future__ import annotations - -import re -from dataclasses import dataclass -from inspect import cleandoc - - -class ParseError(Exception): - """ - Indicates inconsistencies when parsing a changelog from raw - text. E.g. a section with a body but no title. - """ - - -@dataclass -class Section: - title: str - body: str - - @property - def rendered(self) -> str: - return f"{self.title}\n\n{self.body}" - - def replace_prefix(self, prefix: str) -> None: - """ - Prepends the specified prefix to the body of the section. - - If the body starts with the first line of the specified prefix, then - replace the body's prefix. - """ - flags = re.DOTALL | re.MULTILINE - if not self.body.startswith(prefix.splitlines()[0]): - self.body = f"{prefix}\n\n{self.body}" if self.body else prefix - elif re.search(r"^[*-] ", self.body, flags=flags): - suffix = re.sub(r".*?^([*-])", r"\1", self.body, count=1, flags=flags) - self.body = f"{prefix}\n\n{suffix}" - else: - self.body = prefix - - -@dataclass -class ChangesFile: - """ - Represents file unreleased.md or changes_*.py in folder doc/changes/. - """ - - sections: list[Section] - - def get_section(self, title: str) -> Section | None: - """ - Retrieve the section with the specified title. - """ - - pattern = re.compile(f"#+ {re.escape(title)}$") - return next((s for s in self.sections if pattern.match(s.title)), None) - - def add_section(self, section: Section, pos: int = 1) -> None: - """ - Insert the specified section at the specified position. - """ - - self.sections.insert(pos, section) - - @property - def rendered(self) -> str: - return "\n\n".join(s.rendered for s in self.sections) - - @classmethod - def parse(cls, content: str) -> ChangesFile: - title = None - body = [] - sections = [] - - def is_body(line: str) -> bool: - return not line.startswith("#") - - def process_section(): - nonlocal sections - if title: - sections.append(Section(title, cleandoc("\n".join(body)))) - - for line in content.splitlines(): - if is_body(line): - if not title: - raise ParseError(f"Found body line without preceding title: {line}") - body.append(line) - continue - # found new title - process_section() - title = line - body = [] - - process_section() - return ChangesFile(sections) - - -def sample(): - changes = ChangesFile.parse(content) - if section := changes.get_section(title): - section.replace_prefix(body) - else: - changes.add_section(Section(title, body)) diff --git a/exasol/toolbox/util/release/markdown.py b/exasol/toolbox/util/release/markdown.py new file mode 100644 index 000000000..8e5515b1d --- /dev/null +++ b/exasol/toolbox/util/release/markdown.py @@ -0,0 +1,156 @@ +""" +A project's Changelog is expected to list the changes coming with each of +the project's releases. The Changelog contains a file changelog.md with the +table of contents. and zero or more changes files. + +Each changes file is named changes_*.md and describes the changes for a +specific release. The * in the file name equals the version number of the +related release. + +All files are in Markdown syntax, divided into sections. Each section is +identified by its title which should be unique as is represented by class +Markdown. + +A section may consist of a prefix and a suffix, either might be empty. The +prefix are some introductory sentences, the suffix is the list of issues in +this section. Optionally each section can contain subsections as children. + +Method Markdown.replace_prefix() adds such a prefix or replaces it, when the +section already has one. + +The first line of each changes file must be the title describing the version +number, date and name of the release, followed by zero or multiple +sections. The first section is a summary, each other section lists the issues +(aka. tickets) of a particular category the are resolved by this +release. Categories are security, bugfixes, features, documentation, and +refactorings. +""" + +from __future__ import annotations + +import io +import re +from dataclasses import dataclass +from inspect import cleandoc + +from dataclasses import field + + +class ParseError(Exception): + """ + Indicates inconsistencies when parsing a changelog from raw + text. E.g. a section with a body but no title. + """ + + +class HierarchyError(Exception): + """ + When adding a child to a parent with higher level title. + """ + + +def is_title(line: str) -> bool: + return line and line.startswith("#") + + +def is_list_item(line: str) -> bool: + return line and (line.startswith("#") or line.startswith("-")) + + +def is_intro(line: str) -> bool: + return line and not is_title(line) and not is_list_item(line) + + +def level(title: str) -> int: + """ + Return the hierarchical level of the title, i.e. the number of "#" + chars at the beginning of the title. + """ + return len(title) - len(title.lstrip("#")) + + +@dataclass +class Markdown: + """ + Represents a Markdown file or a section within a Markdown file. + """ + + def __init__(self, title: str, intro: str, items: str, children: list[Markdown]): + self.title = title.rstrip("\n") + self.intro = intro + self.items = items + self.children = children + + def can_contain(self, child: Markdown) -> bool: + return level(self.title) < level(child.title) + + def child(self, title: str) -> Markdown | None: + """ + Retrieve the child with the specified title. + """ + + pattern = re.compile(f"#+ {re.escape(title)}$") + return next((c for c in self.children if pattern.match(c.title)), None) + + def add_child(self, child: Markdown, pos: int = 1) -> None: + """ + Insert the specified section as child at the specified position. + """ + + if not self.can_contain(child): + raise HierarchyError( + f'Markdown section "{self.title}" cannot have "{child.title}" as child.' + ) + self.children.insert(pos, child) + + @property + def rendered(self) -> str: + def elements(): + yield from (self.title, self.intro, self.items) + yield from (c.rendered for c in self.children) + + return "\n\n".join(e for e in elements() if e) + + @classmethod + def parse(cls, content: str) -> Markdown: + stream = io.StringIO(content) + line = stream.readline() + if not is_title(line): + raise ParseError( + f'First line of markdown file must be a title, but is "{line}"' + ) + + section, line = cls._parse(stream, line) + if not line: + return section + raise ParseError( + f'Found additional line "{line}" after top-level section "{section.title}".' + ) + + @classmethod + def _parse(cls, stream: io.TextIOBase, title: str) -> tuple[Markdown, str]: + intro = "" + items = "" + children = [] + + line = stream.readline() + while is_intro(line): + intro += line + line = stream.readline() + if is_list_item(line): + while not is_title(line): + items += line + line = stream.readline() + while is_title(line) and level(title) < level(line): + child, line = Markdown._parse(stream, title=line) + children.append(child) + return Markdown(title, intro.strip("\n"), items.strip("\n"), children), line + +def sample(): + changes = Markdown.parse(content) + resolved_vulnerabilities = "" + intro = resolved_vulnerabilities + if section := changes.child(title): + section.intro = intro + else: + changes.add_child(Section(title, intro, items="", [])) diff --git a/test/unit/util/release/test_changes_file.py b/test/unit/util/release/test_changes_file.py deleted file mode 100644 index 2d5f80dd5..000000000 --- a/test/unit/util/release/test_changes_file.py +++ /dev/null @@ -1,120 +0,0 @@ -from inspect import cleandoc - -import pytest - -from exasol.toolbox.util.release.changes_file import ( - ChangesFile, - ParseError, - Section, -) - -import pytest - - -class Scenario: - def __init__(self, initial: str, expected_output: str, expected_sections: list[str]): - self.testee = ChangesFile.parse(cleandoc(initial)) - self.expected_output = cleandoc(expected_output) - self.expected_sections = expected_sections - - -EMPTY = Scenario( - initial="", - expected_output="", - expected_sections=[], -) - -MINIMAL = Scenario( - initial=""" - # title - body - """, - expected_output=""" - # title - - body - """, - expected_sections=["title"], -) - -SPECIAL_CHAR_TITLE = "+ special [char] * title" - -SPECIAL_CHAR_SECTION = Scenario( - initial=f""" - # {SPECIAL_CHAR_TITLE} - body - """, - expected_output=""" - # {SPECIAL_CHAR_TITLE} - - body - """, - expected_sections=[SPECIAL_CHAR_TITLE], -) - -WITH_SUBSECTION = Scenario( - initial=""" - # title - body - ## subtitle - paragraph - - * item 1 - * item 2 - """, - expected_output=""" - # title - - body - - ## subtitle - - paragraph - - * item 1 - * item 2 - """, - expected_sections=["title","subtitle"] - ) - - -def test_parse_error() -> None: - with pytest.raises(ParseError, match="Found body line without preceding title"): - ChangesFile.parse("body line") - - -@pytest.mark.parametrize("scenario", [EMPTY, MINIMAL, WITH_SUBSECTION]) -def test_number_of_sections(scenario: Scenario): - assert len(scenario.testee.sections) == len(scenario.expected_sections) - - -@pytest.mark.parametrize("scenario", [EMPTY, MINIMAL, SPECIAL_CHAR_SECTION, WITH_SUBSECTION]) -def test_get(scenario: Scenario): - assert all(scenario.testee.get_section(s) for s in scenario.expected_sections) - - -@pytest.mark.parametrize("scenario", [EMPTY, MINIMAL, WITH_SUBSECTION]) -def test_missing_section(scenario: Scenario): - assert scenario.testee.get_section("non existing") is None - - -@pytest.mark.parametrize("scenario", [EMPTY, MINIMAL, WITH_SUBSECTION]) -def test_render(scenario: Scenario): - assert scenario.testee.rendered == scenario.expected_output - - -@pytest.fixture -def sample_section(): - return Section("# blabla", "body") - - -@pytest.mark.parametrize("scenario", [MINIMAL, WITH_SUBSECTION]) -def test_add_non_empty(scenario: Scenario, sample_section): - scenario.testee.add_section(sample_section) - assert scenario.testee.sections[1] == sample_section - - -@pytest.mark.parametrize("scenario", [EMPTY]) -def test_add_empty(scenario: Scenario, sample_section): - scenario.testee.add_section(sample_section) - assert scenario.testee.sections[0] == sample_section diff --git a/test/unit/util/release/test_markdown.py b/test/unit/util/release/test_markdown.py new file mode 100644 index 000000000..6fdca49e6 --- /dev/null +++ b/test/unit/util/release/test_markdown.py @@ -0,0 +1,216 @@ +from inspect import cleandoc + +import pytest + +from exasol.toolbox.util.release.markdown import ( + HierarchyError, + Markdown, + ParseError, +) + + +class Scenario: + def __init__( + self, initial: str, expected_output: str, expected_children: list[str] + ): + self.initial = cleandoc(initial) + self.expected_output = cleandoc(expected_output) + self.expected_children = expected_children + + def create_testee(self) -> Markdown: + return Markdown.parse(self.initial) + + +INVALID_MARKDOWN = cleandoc(""" + # Title + + Some text. + + # Another Title + """) + +MINIMAL = Scenario( + initial=""" + # title + body + """, + expected_output=""" + # title + + body + """, + expected_children=[], +) + +WITH_CHILD = Scenario( + initial=""" + # Parent + text + ## Child + paragraph + + * item 1 + * item 2 + """, + expected_output=""" + # Parent + + text + + ## Child + + paragraph + + * item 1 + * item 2 + """, + expected_children=["Child"], +) + +TWO_CHILDREN = Scenario( + initial=""" + # Parent + text + ## C1 + aaa + ## C2 + bbb + """, + expected_output=""" + # Parent + + text + + ## C1 + + aaa + + ## C2 + + bbb + """, + expected_children=["C1", "C2"], +) + + +NESTED = Scenario( + initial=""" + # Parent + text + ## Child A + aaa + ### Grand Child + ccc + ## Child B + bbb + """, + expected_output=""" + # Parent + + text + + ## Child A + + aaa + + ### Grand Child + + ccc + + ## Child B + + bbb + """, + expected_children=["Child A", "Child B"], +) + + +SPECIAL_CHAR_TITLE = "+ special [char] * title" +SPECIAL_CHAR_CHILD = Scenario( + initial=f""" + # title + + ## {SPECIAL_CHAR_TITLE} + body + """, + expected_output=f""" + # title + + ## {SPECIAL_CHAR_TITLE} + + body + """, + expected_children=[SPECIAL_CHAR_TITLE], +) + + +def test_no_title_error(): + with pytest.raises(ParseError, match="First line of markdown file must be a title"): + Markdown.parse("body\n# title") + + +def test_additional_line_error(): + expected_error = ( + 'additional line "# Another Title" after top-level section "# Title".' + ) + with pytest.raises(ParseError, match=expected_error): + Markdown.parse(INVALID_MARKDOWN) + + +ALL_SCENARIOS = [MINIMAL, WITH_CHILD, TWO_CHILDREN, NESTED, SPECIAL_CHAR_CHILD] + + +@pytest.mark.parametrize("scenario", ALL_SCENARIOS) +def test_number_of_children(scenario: Scenario): + assert len(scenario.create_testee().children) == len(scenario.expected_children) + + +@pytest.mark.parametrize("scenario", ALL_SCENARIOS) +def test_non_existing_child(scenario: Scenario): + assert scenario.create_testee().child("non existing") is None + + +@pytest.mark.parametrize("scenario", ALL_SCENARIOS) +def test_valid_child(scenario: Scenario): + assert all(scenario.create_testee().child(c) for c in scenario.expected_children) + + +@pytest.mark.parametrize("scenario", ALL_SCENARIOS) +def test_rendered(scenario: Scenario): + assert scenario.create_testee().rendered == scenario.expected_output + + +@pytest.fixture +def sample_child() -> Markdown: + return Markdown(title="## New", intro="intro", items="", children=[]) + + +@pytest.mark.parametrize( + "scenario, pos", + [ + (MINIMAL, 0), + (WITH_CHILD, 1), + (TWO_CHILDREN, 1), + ], +) +def test_add_child(sample_child: Markdown, scenario: Scenario, pos: int): + testee = scenario.create_testee() + testee.add_child(sample_child) + assert testee.children[pos] == sample_child + + +@pytest.fixture +def illegal_child() -> Markdown: + return Markdown(title="# Top-level", intro="intro", items="", children=[]) + + +@pytest.mark.parametrize("scenario", ALL_SCENARIOS) +def test_illegal_child(illegal_child: Markdown, scenario: Scenario): + testee = scenario.create_testee() + with pytest.raises(HierarchyError): + testee.add_child(illegal_child) + + +def test_nested(): + testee = NESTED.create_testee() + assert testee.child("Child A").child("Grand Child") is not None diff --git a/test/unit/util/release/test_section.py b/test/unit/util/release/test_section.py deleted file mode 100644 index b12e9d28f..000000000 --- a/test/unit/util/release/test_section.py +++ /dev/null @@ -1,84 +0,0 @@ -from inspect import cleandoc - -import pytest - -from exasol.toolbox.util.release.changes_file import Section - - -class Scenario: - def __init__(self, body: str, expected_suffix: str): - self.body = cleandoc(body) - self.expected_suffix = ( - f"\n\n{cleandoc(expected_suffix)}" if expected_suffix else "" - ) - - def create_testee(self) -> Section: - return Section("# title", self.body) - - -NO_MATCHING_PREFIX = Scenario("body", expected_suffix="body") - -MATCHING_PREFIX_BUT_NO_LIST = Scenario( - body=""" - Prefix first line - - Another line - """, - expected_suffix="", -) - -MATCHING_PREFIX_AND_LIST = Scenario( - body=""" - Prefix first line - - Another line - - * item 1 - * item 2 - """, - expected_suffix=""" - * item 1 - * item 2 - """, -) - - -LIST_WITH_DASHES = Scenario( - body=""" - Prefix first line - - Another line - - - item 1 - - item 2 - """, - expected_suffix=""" - - item 1 - - item 2 - """, -) - - -SAMPLE_PREFIX = cleandoc(""" - Prefix first line - - | col 1 | col 2 | - |-------|-------| - | abc | 123 | - """) - - -@pytest.mark.parametrize( - "scenario", - [ - pytest.param(NO_MATCHING_PREFIX, id="no_matching_prefix"), - pytest.param(MATCHING_PREFIX_BUT_NO_LIST, id="matching_prefix_but_no_list"), - pytest.param(MATCHING_PREFIX_AND_LIST, id="matching_prefix_and_list"), - pytest.param(LIST_WITH_DASHES, id="list_with_dashes"), - ], -) -def test_replace_prefix(scenario): - testee = scenario.create_testee() - testee.replace_prefix(SAMPLE_PREFIX) - expected = f"{SAMPLE_PREFIX}{scenario.expected_suffix}" - assert testee.body == f"{SAMPLE_PREFIX}{scenario.expected_suffix}" From 84b75e7b6cf0e1091136ea432f372f5ece039282 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 8 Apr 2026 11:00:12 +0200 Subject: [PATCH 04/37] First steps towards using Markdown --- exasol/toolbox/util/release/changelog.py | 76 ++++++++++++++++-------- 1 file changed, 50 insertions(+), 26 deletions(-) diff --git a/exasol/toolbox/util/release/changelog.py b/exasol/toolbox/util/release/changelog.py index 35027536f..472d17502 100644 --- a/exasol/toolbox/util/release/changelog.py +++ b/exasol/toolbox/util/release/changelog.py @@ -12,6 +12,7 @@ ) from exasol.toolbox.util.dependencies.shared_models import LatestTagNotFoundError from exasol.toolbox.util.dependencies.track_changes import DependencyChanges +from exasol.toolbox.util.release.markdown import Markdown from exasol.toolbox.util.version import Version UNRELEASED_INITIAL_CONTENT = cleandoc(""" @@ -23,7 +24,6 @@ DEPENDENCY_UPDATES = "## Dependency Updates\n" TITLE = "Dependency Updates" -HEADING = "## {TITLE}\n" class Changelogs: @@ -50,7 +50,12 @@ def _create_new_unreleased(self): self.unreleased_md.write_text(UNRELEASED_INITIAL_CONTENT) - def _create_versioned_changelog(self, unreleased_content: str) -> None: + def _dependency_changes(self) -> Markdown | None: + if sections := list(self._dependency_changes()): + return Markdown(f"## {TITLE}", intro="", items="", sections) + return None + + def _create_versioned_changes(self, unreleased_content: str) -> None: """ Create a versioned changes file. @@ -58,23 +63,38 @@ def _create_versioned_changelog(self, unreleased_content: str) -> None: unreleased_content: the content of the (not yet versioned) changes """ - header = f"# {self.version} - {datetime.today().strftime('%Y-%m-%d')}" - dependency_changes = self._report_dependency_changes() - template = cleandoc(f"{header}\n\n{unreleased_content}\n{dependency_changes}") - self.versioned_changelog_md.write_text(template + "\n") - - def _extract_unreleased_notes(self) -> str: - """ - Extract (not yet versioned) changes from `unreleased.md`. - """ - - with self.unreleased_md.open(mode="r", encoding="utf-8") as f: - # skip header when reading in file, as contains # Unreleased - lines = f.readlines()[1:] - unreleased_content = cleandoc("".join(lines)) - return unreleased_content + "\n" - - def _dependency_changes(self) -> str: + versioned = Markdown.parse(unreleased_content) + versioned.title = f"# {self.version} - {datetime.today().strftime('%Y-%m-%d')}" + if section := self._dependency_section(): + versioned.add_child(section) + self.versioned_changelog_md.write_text(versioned.rendered) + + # def _create_versioned_changelog(self, unreleased_content: str) -> None: + # """ + # Create a versioned changes file. + # + # Args: + # unreleased_content: the content of the (not yet versioned) changes + # """ + # + # header = f"# {self.version} - {datetime.today().strftime('%Y-%m-%d')}" + # dependency_changes = self._report_dependency_changes() + # template = cleandoc(f"{header}\n\n{unreleased_content}\n{dependency_changes}") + # self.versioned_changelog_md.write_text(template + "\n") + + # def _extract_unreleased_notes(self) -> str: + # """ + # Extract (not yet versioned) changes from `unreleased.md`. + # """ + # + # with self.unreleased_md.open(mode="r", encoding="utf-8") as f: + # # skip header when reading in file, as contains # Unreleased + # lines = f.readlines()[1:] + # unreleased_content = cleandoc("".join(lines)) + # return unreleased_content + "\n" + + # former: _dependency_changes + def _dependency_sections(self) -> Generator[Markdown]: # str: """ Return the dependency changes between the latest tag and the current version for use in the versioned changes file in markdown @@ -94,8 +114,8 @@ def _dependency_changes(self) -> str: working_directory=self.root_path ) - changes_by_group: list[str] = [] - # dict.keys() returns a set + ## former: + ## changes_by_group: list[str] = [] all_groups = ( previous_dependencies_in_groups.keys() | current_dependencies_in_groups.keys() @@ -109,8 +129,9 @@ def _dependency_changes(self) -> str: ).changes if changes: changes_str = "\n".join(str(change) for change in changes) - changes_by_group.append(f"\n### `{group}`\n\n{changes_str}\n") - return "".join(changes_by_group) + yield Markdown(f"### `{group}`", items=changes_str) + # changes_by_group.append(f"\n### `{group}`\n\n{changes_str}\n") + ## return "".join(changes_by_group) @staticmethod def _sort_groups(groups: set[str]) -> list[str]: @@ -176,9 +197,12 @@ def prepare_release(self) -> Changelogs: 3. Updates the table of contents in the `changelog.md` with the new `changes_.md` """ - # create versioned changelog - unreleased_content = self._extract_unreleased_notes() - self._create_versioned_changelog(unreleased_content) + unreleased = self.unreleased_md.read_text() + self._create_versioned_changes(unreleased) + + # # create versioned changelog + # unreleased_content = self._extract_unreleased_notes() + # self._create_versioned_changelog(unreleased_content) # update other changelogs now that versioned changelog exists self._create_new_unreleased() From e6fc28df23bd81471a34af08db91225f2e82525d Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 8 Apr 2026 12:06:34 +0200 Subject: [PATCH 05/37] merged latest changes from changes-file (2) --- exasol/toolbox/util/release/markdown.py | 61 ++++++++++++++------ test/unit/util/release/test_markdown.py | 74 ++++++++++++++----------- 2 files changed, 88 insertions(+), 47 deletions(-) diff --git a/exasol/toolbox/util/release/markdown.py b/exasol/toolbox/util/release/markdown.py index 8e5515b1d..7496dbc73 100644 --- a/exasol/toolbox/util/release/markdown.py +++ b/exasol/toolbox/util/release/markdown.py @@ -29,11 +29,6 @@ from __future__ import annotations import io -import re -from dataclasses import dataclass -from inspect import cleandoc - -from dataclasses import field class ParseError(Exception): @@ -43,7 +38,7 @@ class ParseError(Exception): """ -class HierarchyError(Exception): +class IllegalChild(Exception): """ When adding a child to a parent with higher level title. """ @@ -69,39 +64,70 @@ def level(title: str) -> int: return len(title) - len(title.lstrip("#")) -@dataclass class Markdown: """ Represents a Markdown file or a section within a Markdown file. """ - def __init__(self, title: str, intro: str, items: str, children: list[Markdown]): + def __init__( + self, + title: str, + intro: str = "", + items: str = "", + children: list[Markdown] | None = None, + ): self.title = title.rstrip("\n") self.intro = intro self.items = items + children = children or [] + for child in children: + self._check(child) self.children = children def can_contain(self, child: Markdown) -> bool: return level(self.title) < level(child.title) + def find(self, child_title: str) -> tuple[int, Markdown] | None: + """ + Return index and child having the specified title, or None if + there is none. + """ + for i, child in enumerate(self.children): + if child.title == child_title: + return i, child + return None + def child(self, title: str) -> Markdown | None: """ Retrieve the child with the specified title. """ + return found[1] if (found := self.find(title)) else None - pattern = re.compile(f"#+ {re.escape(title)}$") - return next((c for c in self.children if pattern.match(c.title)), None) + def _check(self, child: Markdown) -> Markdown: + if not self.can_contain(child): + raise IllegalChild( + f'Markdown section "{self.title}" cannot have "{child.title}" as child.' + ) + return child def add_child(self, child: Markdown, pos: int = 1) -> None: """ Insert the specified section as child at the specified position. """ - if not self.can_contain(child): - raise HierarchyError( - f'Markdown section "{self.title}" cannot have "{child.title}" as child.' - ) - self.children.insert(pos, child) + self.children.insert(pos, self._check(child)) + + def replace_child(self, child: Markdown) -> None: + """ + If there is a child with the same title then replace this child + otherwise append the specified child. + """ + + self._check(child) + if found := self.find(child.title): + self.children[found[0]] = child + else: + self.children.append(child) @property def rendered(self) -> str: @@ -146,11 +172,14 @@ def _parse(cls, stream: io.TextIOBase, title: str) -> tuple[Markdown, str]: children.append(child) return Markdown(title, intro.strip("\n"), items.strip("\n"), children), line + def sample(): + content = "" changes = Markdown.parse(content) resolved_vulnerabilities = "" intro = resolved_vulnerabilities + title = "# title" if section := changes.child(title): section.intro = intro else: - changes.add_child(Section(title, intro, items="", [])) + changes.add_child(Markdown(title, intro)) diff --git a/test/unit/util/release/test_markdown.py b/test/unit/util/release/test_markdown.py index 6fdca49e6..24013f111 100644 --- a/test/unit/util/release/test_markdown.py +++ b/test/unit/util/release/test_markdown.py @@ -3,7 +3,7 @@ import pytest from exasol.toolbox.util.release.markdown import ( - HierarchyError, + IllegalChild, Markdown, ParseError, ) @@ -64,7 +64,7 @@ def create_testee(self) -> Markdown: * item 1 * item 2 """, - expected_children=["Child"], + expected_children=["## Child"], ) TWO_CHILDREN = Scenario( @@ -89,7 +89,7 @@ def create_testee(self) -> Markdown: bbb """, - expected_children=["C1", "C2"], + expected_children=["## C1", "## C2"], ) @@ -121,27 +121,18 @@ def create_testee(self) -> Markdown: bbb """, - expected_children=["Child A", "Child B"], + expected_children=["## Child A", "## Child B"], ) -SPECIAL_CHAR_TITLE = "+ special [char] * title" -SPECIAL_CHAR_CHILD = Scenario( - initial=f""" - # title - - ## {SPECIAL_CHAR_TITLE} - body - """, - expected_output=f""" - # title +@pytest.fixture +def sample_child() -> Markdown: + return Markdown(title="## New", intro="intro") - ## {SPECIAL_CHAR_TITLE} - body - """, - expected_children=[SPECIAL_CHAR_TITLE], -) +@pytest.fixture +def illegal_child() -> Markdown: + return Markdown(title="# Top-level", intro="intro") def test_no_title_error(): @@ -157,7 +148,12 @@ def test_additional_line_error(): Markdown.parse(INVALID_MARKDOWN) -ALL_SCENARIOS = [MINIMAL, WITH_CHILD, TWO_CHILDREN, NESTED, SPECIAL_CHAR_CHILD] +def test_constructor_illegal_child(illegal_child: Markdown): + with pytest.raises(IllegalChild): + Markdown("# title", children=[illegal_child]) + + +ALL_SCENARIOS = [MINIMAL, WITH_CHILD, TWO_CHILDREN, NESTED] @pytest.mark.parametrize("scenario", ALL_SCENARIOS) @@ -180,11 +176,6 @@ def test_rendered(scenario: Scenario): assert scenario.create_testee().rendered == scenario.expected_output -@pytest.fixture -def sample_child() -> Markdown: - return Markdown(title="## New", intro="intro", items="", children=[]) - - @pytest.mark.parametrize( "scenario, pos", [ @@ -199,18 +190,39 @@ def test_add_child(sample_child: Markdown, scenario: Scenario, pos: int): assert testee.children[pos] == sample_child -@pytest.fixture -def illegal_child() -> Markdown: - return Markdown(title="# Top-level", intro="intro", items="", children=[]) +def test_replace_illegal_child(illegal_child): + testee = WITH_CHILD.create_testee() + with pytest.raises(IllegalChild): + testee.replace_child(illegal_child) + + +@pytest.mark.parametrize("scenario", ALL_SCENARIOS) +def test_replace_existing_child(scenario: Scenario): + testee = WITH_CHILD.create_testee() + old_child = testee.children[0] + old_rendered = testee.rendered + new_child = Markdown(old_child.title, "new intro") + expected = old_rendered.replace(old_child.rendered, new_child.rendered) + testee.replace_child(new_child) + assert testee.rendered == expected + + +@pytest.mark.parametrize("scenario", ALL_SCENARIOS) +def test_replace_non_existing_child(scenario: Scenario, sample_child: Markdown): + testee = scenario.create_testee() + expected = len(testee.children) + 1 + testee.replace_child(sample_child) + assert len(testee.children) == expected + assert testee.children[-1] == sample_child @pytest.mark.parametrize("scenario", ALL_SCENARIOS) -def test_illegal_child(illegal_child: Markdown, scenario: Scenario): +def test_add_illegal_child(illegal_child: Markdown, scenario: Scenario): testee = scenario.create_testee() - with pytest.raises(HierarchyError): + with pytest.raises(IllegalChild): testee.add_child(illegal_child) def test_nested(): testee = NESTED.create_testee() - assert testee.child("Child A").child("Grand Child") is not None + assert testee.child("## Child A").child("### Grand Child") is not None From d0610ec7d36d79dc4da2f5804e523372e1cfcf83 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 8 Apr 2026 12:35:31 +0200 Subject: [PATCH 06/37] Finalized implementation (vulnerabilities currently commented-out, yet) --- .../dependencies/track_vulnerabilities.py | 3 +- exasol/toolbox/util/release/changelog.py | 135 +++++++----------- test/unit/nox/_dependencies_test.py | 4 +- test/unit/util/release/changelog_test.py | 2 +- 4 files changed, 53 insertions(+), 91 deletions(-) diff --git a/exasol/toolbox/util/dependencies/track_vulnerabilities.py b/exasol/toolbox/util/dependencies/track_vulnerabilities.py index ec079999c..38e4436d6 100644 --- a/exasol/toolbox/util/dependencies/track_vulnerabilities.py +++ b/exasol/toolbox/util/dependencies/track_vulnerabilities.py @@ -66,9 +66,8 @@ def report_resolved_vulnerabilities(self) -> str: if not (resolved := self.resolved_vulnerabilities): return "" header = cleandoc( - """ ## Fixed Vulnerabilities - + """ This release fixes vulnerabilities by updating dependencies: | Dependency | Vulnerability | Affected | Fixed in | diff --git a/exasol/toolbox/util/release/changelog.py b/exasol/toolbox/util/release/changelog.py index 472d17502..b1894310f 100644 --- a/exasol/toolbox/util/release/changelog.py +++ b/exasol/toolbox/util/release/changelog.py @@ -21,10 +21,6 @@ ## Summary """) + "\n" -DEPENDENCY_UPDATES = "## Dependency Updates\n" - -TITLE = "Dependency Updates" - class Changelogs: def __init__(self, changes_path: Path, root_path: Path, version: Version) -> None: @@ -52,49 +48,10 @@ def _create_new_unreleased(self): def _dependency_changes(self) -> Markdown | None: if sections := list(self._dependency_changes()): - return Markdown(f"## {TITLE}", intro="", items="", sections) + return Markdown(f"## Dependency Updates", children=sections) return None - def _create_versioned_changes(self, unreleased_content: str) -> None: - """ - Create a versioned changes file. - - Args: - unreleased_content: the content of the (not yet versioned) changes - """ - - versioned = Markdown.parse(unreleased_content) - versioned.title = f"# {self.version} - {datetime.today().strftime('%Y-%m-%d')}" - if section := self._dependency_section(): - versioned.add_child(section) - self.versioned_changelog_md.write_text(versioned.rendered) - - # def _create_versioned_changelog(self, unreleased_content: str) -> None: - # """ - # Create a versioned changes file. - # - # Args: - # unreleased_content: the content of the (not yet versioned) changes - # """ - # - # header = f"# {self.version} - {datetime.today().strftime('%Y-%m-%d')}" - # dependency_changes = self._report_dependency_changes() - # template = cleandoc(f"{header}\n\n{unreleased_content}\n{dependency_changes}") - # self.versioned_changelog_md.write_text(template + "\n") - - # def _extract_unreleased_notes(self) -> str: - # """ - # Extract (not yet versioned) changes from `unreleased.md`. - # """ - # - # with self.unreleased_md.open(mode="r", encoding="utf-8") as f: - # # skip header when reading in file, as contains # Unreleased - # lines = f.readlines()[1:] - # unreleased_content = cleandoc("".join(lines)) - # return unreleased_content + "\n" - - # former: _dependency_changes - def _dependency_sections(self) -> Generator[Markdown]: # str: + def _dependency_sections(self) -> Generator[Markdown]: """ Return the dependency changes between the latest tag and the current version for use in the versioned changes file in markdown @@ -102,36 +59,26 @@ def _dependency_sections(self) -> Generator[Markdown]: # str: """ try: - previous_dependencies_in_groups = get_dependencies_from_latest_tag( + previous_groups = get_dependencies_from_latest_tag( root_path=self.root_path ) except LatestTagNotFoundError: # In new projects, there is not a pre-existing tag, and all dependencies # are considered new. - previous_dependencies_in_groups = OrderedDict() - - current_dependencies_in_groups = get_dependencies( - working_directory=self.root_path - ) - - ## former: - ## changes_by_group: list[str] = [] - all_groups = ( - previous_dependencies_in_groups.keys() - | current_dependencies_in_groups.keys() - ) + previous_groups = OrderedDict() + + current_groups = get_dependencies(working_directory=self.root_path) + all_groups = previous_groups.keys() | current_groups.keys() + for group in self._sort_groups(all_groups): - previous_dependencies = previous_dependencies_in_groups.get(group, {}) - current_dependencies = current_dependencies_in_groups.get(group, {}) - changes = DependencyChanges( - previous_dependencies=previous_dependencies, - current_dependencies=current_dependencies, - ).changes - if changes: - changes_str = "\n".join(str(change) for change in changes) - yield Markdown(f"### `{group}`", items=changes_str) - # changes_by_group.append(f"\n### `{group}`\n\n{changes_str}\n") - ## return "".join(changes_by_group) + previous = previous_groups.get(group, {}) + current = current_groups.get(group, {}) + if changes := DependencyChanges( + previous_dependencies=previous, + current_dependencies=current, + ).changes: + items = "\n".join(str(change) for change in changes) + yield Markdown(f"### `{group}`", items=items) @staticmethod def _sort_groups(groups: set[str]) -> list[str]: @@ -171,22 +118,31 @@ def _update_table_of_contents(self) -> None: def get_changed_files(self) -> list[Path]: return [self.unreleased_md, self.versioned_changelog_md, self.changelog_md] - def _report_dependency_changes(self) -> str: - if changes := self._dependency_changes(): - return f"## {TITLE}\n{changes}" - return "" + def _resolved_vulnerabilities(self) -> Markdown | None: + report = DependenciesAudit( + previous_vulnerabilities=get_vulnerabilities_from_latest_tag(path), + current_vulnerabilities=get_vulnerabilities(path), + ).report_resolved_vulnerabilities() + return Markdown("## Security Issues", report) if report else None - def update_latest(self) -> Changelogs: + def _create_versioned_changes(self, initial_content: str) -> None: """ - Update the updated dependencies in the latest versioned changelog. + Create a versioned changes file. + + Args: + unreleased_content: the content of the (not yet versioned) changes """ - content = self.versioned_changelog_md.read_text() - flags = re.DOTALL | re.MULTILINE - stripped = re.sub(r"^{DEPENDENCY_UPDATES}.*", "", content, flags=flags) - dependency_changes = self._report_dependency_changes() - self.versioned_changelog_md.write_text(f"{stripped}\n{dependency_changes}") - return self + versioned = Markdown.parse(initial_content) + versioned.title = f"# {self.version} - {datetime.today().strftime('%Y-%m-%d')}" + if dependency_changes := self._dependency_changes(): + versioned.replace_child(dependency_changes) + # if resolved_vulnerabilities := self._resolved_vulnerabilities(): + # if section := versioned.child(resolved_vulnerabilities.title): + # section.intro = resolved_vulnerabilities + # else: + # versioned.add_child(resolved_vulnerabilities) + self.versioned_changelog_md.write_text(versioned.rendered) def prepare_release(self) -> Changelogs: """ @@ -197,14 +153,19 @@ def prepare_release(self) -> Changelogs: 3. Updates the table of contents in the `changelog.md` with the new `changes_.md` """ - unreleased = self.unreleased_md.read_text() - self._create_versioned_changes(unreleased) - - # # create versioned changelog - # unreleased_content = self._extract_unreleased_notes() - # self._create_versioned_changelog(unreleased_content) + content = self.unreleased_md.read_text() + self._create_versioned_changes(content) # update other changelogs now that versioned changelog exists self._create_new_unreleased() self._update_table_of_contents() return self + + def update_latest(self) -> Changelogs: + """ + Update the updated dependencies in the latest versioned changelog. + """ + + content = self.versioned_changelog_md.read_text() + self._create_versioned_changes(content) + return self diff --git a/test/unit/nox/_dependencies_test.py b/test/unit/nox/_dependencies_test.py index a391193e4..dd6e0985a 100644 --- a/test/unit/nox/_dependencies_test.py +++ b/test/unit/nox/_dependencies_test.py @@ -14,7 +14,9 @@ def test_audit(monkeypatch, nox_session, sample_vulnerability, capsys): assert capsys.readouterr().out == sample_vulnerability.nox_dependencies_audit -def test_report_resolved_vulnerabilities(monkeypatch, nox_session, capsys, sample_vulnerability): +def test_report_resolved_vulnerabilities( + monkeypatch, nox_session, capsys, sample_vulnerability +): monkeypatch.setattr( _dependencies, "get_vulnerabilities_from_latest_tag", diff --git a/test/unit/util/release/changelog_test.py b/test/unit/util/release/changelog_test.py index f2c5c5de1..bed036a76 100644 --- a/test/unit/util/release/changelog_test.py +++ b/test/unit/util/release/changelog_test.py @@ -152,7 +152,7 @@ def test_create_new_unreleased(changelogs): @staticmethod def test_create_versioned_changelog(changelogs, mock_dependencies): - changelogs._create_versioned_changelog(SampleContent.changelog) + changelogs._create_versioned_changes(SampleContent.changelog) saved_text = changelogs.versioned_changelog_md.read_text() assert "1.0.0" in saved_text From dcb7428337f8ccff9002a91954042a4f8fbc30f5 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 8 Apr 2026 17:03:19 +0200 Subject: [PATCH 07/37] Fixed and refactored tests --- exasol/toolbox/nox/_release.py | 6 +- exasol/toolbox/util/release/changelog.py | 36 ++-- exasol/toolbox/util/release/markdown.py | 10 +- test/unit/util/release/changelog_test.py | 220 ++++++++++++----------- 4 files changed, 144 insertions(+), 128 deletions(-) diff --git a/exasol/toolbox/nox/_release.py b/exasol/toolbox/nox/_release.py index 27d25647e..160729220 100644 --- a/exasol/toolbox/nox/_release.py +++ b/exasol/toolbox/nox/_release.py @@ -15,7 +15,7 @@ from exasol.toolbox.nox.plugin import NoxTasks from exasol.toolbox.util.dependencies.shared_models import PoetryFiles from exasol.toolbox.util.git import Git -from exasol.toolbox.util.release.changelog import Changelogs +from exasol.toolbox.util.release.changelog import Changelog from exasol.toolbox.util.version import ( ReleaseTypes, Version, @@ -64,8 +64,8 @@ def _update_project_version(session: Session, version: Version) -> Version: return version -def _get_changelogs(version: Version) -> Changelogs: - return Changelogs( +def _get_changelogs(version: Version) -> Changelog: + return Changelog( changes_path=PROJECT_CONFIG.documentation_path / "changes", root_path=PROJECT_CONFIG.root_path, version=version, diff --git a/exasol/toolbox/util/release/changelog.py b/exasol/toolbox/util/release/changelog.py index b1894310f..8f6dc3f27 100644 --- a/exasol/toolbox/util/release/changelog.py +++ b/exasol/toolbox/util/release/changelog.py @@ -22,7 +22,7 @@ """) + "\n" -class Changelogs: +class Changelog: def __init__(self, changes_path: Path, root_path: Path, version: Version) -> None: """ Args: @@ -34,9 +34,9 @@ def __init__(self, changes_path: Path, root_path: Path, version: Version) -> Non """ self.version = version - self.unreleased_md: Path = changes_path / "unreleased.md" - self.versioned_changelog_md: Path = changes_path / f"changes_{version}.md" - self.changelog_md: Path = changes_path / "changelog.md" + self.unreleased: Path = changes_path / "unreleased.md" + self.versioned_changes: Path = changes_path / f"changes_{version}.md" + self.changelog: Path = changes_path / "changelog.md" self.root_path: Path = root_path def _create_new_unreleased(self): @@ -44,12 +44,7 @@ def _create_new_unreleased(self): Write a new unreleased changelog file. """ - self.unreleased_md.write_text(UNRELEASED_INITIAL_CONTENT) - - def _dependency_changes(self) -> Markdown | None: - if sections := list(self._dependency_changes()): - return Markdown(f"## Dependency Updates", children=sections) - return None + self.unreleased.write_text(UNRELEASED_INITIAL_CONTENT) def _dependency_sections(self) -> Generator[Markdown]: """ @@ -80,6 +75,11 @@ def _dependency_sections(self) -> Generator[Markdown]: items = "\n".join(str(change) for change in changes) yield Markdown(f"### `{group}`", items=items) + def _dependency_changes(self) -> Markdown | None: + if sections := list(self._dependency_sections()): + return Markdown(f"## Dependency Updates", children=sections) + return None + @staticmethod def _sort_groups(groups: set[str]) -> list[str]: """ @@ -102,7 +102,7 @@ def _update_table_of_contents(self) -> None: to the relevant sections, and write the updated changelog.md again. """ updated_content = [] - with self.changelog_md.open(mode="r", encoding="utf-8") as f: + with self.changelog.open(mode="r", encoding="utf-8") as f: for line in f: updated_content.append(line) if line.startswith("* [unreleased]"): @@ -113,10 +113,10 @@ def _update_table_of_contents(self) -> None: updated_content.append(f"changes_{self.version}\n") updated_content_str = "".join(updated_content) - self.changelog_md.write_text(updated_content_str) + self.changelog.write_text(updated_content_str) def get_changed_files(self) -> list[Path]: - return [self.unreleased_md, self.versioned_changelog_md, self.changelog_md] + return [self.unreleased, self.versioned_changes, self.changelog] def _resolved_vulnerabilities(self) -> Markdown | None: report = DependenciesAudit( @@ -142,9 +142,9 @@ def _create_versioned_changes(self, initial_content: str) -> None: # section.intro = resolved_vulnerabilities # else: # versioned.add_child(resolved_vulnerabilities) - self.versioned_changelog_md.write_text(versioned.rendered) + self.versioned_changes.write_text(versioned.rendered) - def prepare_release(self) -> Changelogs: + def prepare_release(self) -> Changelog: """ Rotates the changelogs as is needed for a release. @@ -153,7 +153,7 @@ def prepare_release(self) -> Changelogs: 3. Updates the table of contents in the `changelog.md` with the new `changes_.md` """ - content = self.unreleased_md.read_text() + content = self.unreleased.read_text() self._create_versioned_changes(content) # update other changelogs now that versioned changelog exists @@ -161,11 +161,11 @@ def prepare_release(self) -> Changelogs: self._update_table_of_contents() return self - def update_latest(self) -> Changelogs: + def update_latest(self) -> Changelog: """ Update the updated dependencies in the latest versioned changelog. """ - content = self.versioned_changelog_md.read_text() + content = self.versioned_changes.read_text() self._create_versioned_changes(content) return self diff --git a/exasol/toolbox/util/release/markdown.py b/exasol/toolbox/util/release/markdown.py index 7496dbc73..0030ccc83 100644 --- a/exasol/toolbox/util/release/markdown.py +++ b/exasol/toolbox/util/release/markdown.py @@ -117,7 +117,7 @@ def add_child(self, child: Markdown, pos: int = 1) -> None: self.children.insert(pos, self._check(child)) - def replace_child(self, child: Markdown) -> None: + def replace_child(self, child: Markdown) -> Markdown: """ If there is a child with the same title then replace this child otherwise append the specified child. @@ -128,6 +128,7 @@ def replace_child(self, child: Markdown) -> None: self.children[found[0]] = child else: self.children.append(child) + return self @property def rendered(self) -> str: @@ -137,6 +138,13 @@ def elements(): return "\n\n".join(e for e in elements() if e) + def __eq__(self, other) -> bool: + return isinstance(other, Markdown) and self.rendered == other.rendered + + @classmethod + def read(cls, path: Path) -> Markdown: + return cls.parse(path.read_text()) + @classmethod def parse(cls, content: str) -> Markdown: stream = io.StringIO(content) diff --git a/test/unit/util/release/changelog_test.py b/test/unit/util/release/changelog_test.py index bed036a76..7b8bae7c5 100644 --- a/test/unit/util/release/changelog_test.py +++ b/test/unit/util/release/changelog_test.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from datetime import datetime from inspect import cleandoc from unittest.mock import Mock @@ -8,25 +10,39 @@ from exasol.toolbox.util.dependencies.shared_models import LatestTagNotFoundError from exasol.toolbox.util.release.changelog import ( UNRELEASED_INITIAL_CONTENT, - Changelogs, + Changelog, ) +from exasol.toolbox.util.release.markdown import Markdown from exasol.toolbox.util.version import Version -class SampleContent: - changelog = "\n" + cleandoc(""" - Summary of changes. +class SampleData: + def __init__(self, unreleased: str, old_changelog: str, new_changelog: str): + self.unreleased = Markdown.parse(unreleased).rendered + self.unreleased_body = "\n".join(self.unreleased.splitlines()[1:]) + self.old_changelog = old_changelog + self.new_changelog = new_changelog + - ## Added - * Added Awesome feature +SAMPLE = SampleData( + unreleased=cleandoc( + """ + # Unreleased + ## Summary + Summary of changes. - ## Changed - * Some behaviour + ## Features + * Added awesome feature - ## Fixed + ## Bugfixes * Fixed nasty bug - """) - changes = cleandoc(""" + + ## Refactorings + * Some refactoring + """ + ), + old_changelog=cleandoc( + """ # Changelog * [unreleased](unreleased.md) @@ -39,8 +55,10 @@ class SampleContent: unreleased changes_0.1.0 ``` - """) - altered_changes = cleandoc(""" + """ + ), + new_changelog=cleandoc( + """ # Changelog * [unreleased](unreleased.md) @@ -55,50 +73,52 @@ class SampleContent: changes_1.0.0 changes_0.1.0 ``` - """) + """ + ), +) -def expected_changes_file_content(with_dependencies: bool = False): - header = cleandoc(f""" - # 1.0.0 - {datetime.today().strftime('%Y-%m-%d')} +def _markdown(content: str) -> Markdown: + return Markdown.parse(cleandoc(content)) - ## Summary - - Summary of changes. - ## Added - * Added Awesome feature +def expected_changes_file_content(with_dependencies: bool = False) -> Markdown: + changes = Markdown.parse(SAMPLE.unreleased) + changes.title = f"# 1.0.0 - {datetime.today().strftime('%Y-%m-%d')}" + if not with_dependencies: + return changes - ## Changed - * Some behaviour - - ## Fixed - * Fixed nasty bug - """) - dependencies = cleandoc(f""" + dependencies = _markdown( + f""" ## Dependency Updates - ### `main` - * Updated dependency `package1:0.0.1` to `0.1.0` - ### `dev` - * Added dependency `package2:0.2.0` - """) - return f"{header}\n\n{dependencies}\n" if with_dependencies else f"{header}\n" + """ + ) + return changes.replace_child(dependencies) + + +@pytest.fixture(scope="function") +def changelog(tmp_path) -> Changelog: + changes_path = tmp_path / "doc/changes" + changes_path.mkdir(parents=True) + return Changelog( + changes_path=changes_path, + root_path=tmp_path, + version=Version(major=1, minor=0, patch=0), + ) @pytest.fixture(scope="function") -def changes_md(changelogs): - changelogs.changelog_md.write_text(SampleContent.changes) +def changes_md(changelog): + changelog.changelog.write_text(SAMPLE.old_changelog) @pytest.fixture(scope="function") -def unreleased_md(changelogs): - changelogs.unreleased_md.write_text( - UNRELEASED_INITIAL_CONTENT + SampleContent.changelog - ) +def unreleased_md(changelog): + changelog.unreleased.write_text(SAMPLE.unreleased) def mock_changelog(monkeypatch, old_dependencies, new_dependencies): @@ -125,70 +145,58 @@ def mock_no_dependencies(monkeypatch): mock_changelog(monkeypatch, {}, {}) -@pytest.fixture(scope="function") -def changelogs(tmp_path) -> Changelogs: - changes_path = tmp_path / "doc/changes" - changes_path.mkdir(parents=True) - return Changelogs( - changes_path=changes_path, - root_path=tmp_path, - version=Version(major=1, minor=0, patch=0), - ) - - class TestChangelogs: """ - As some methods in the class `Changelogs` modify files, it is required that the - fixtures which create the sample files (changelog.md, unreleased.md, & changes_1.0.0.md) - reset per function and use `tmp_path`. By doing this, we ensure that the sample - are in their expected state for each test. + As some methods in the class `Changelog` modify files, it is required + that the fixtures which create the sample files (changelog.md, + unreleased.md, & changes_1.0.0.md) reset per function and use + `tmp_path`. By doing this, we ensure that the sample are in their expected + state for each test. """ @staticmethod - def test_create_new_unreleased(changelogs): - changelogs._create_new_unreleased() + def test_create_new_unreleased(changelog): + changelog._create_new_unreleased() - assert changelogs.unreleased_md.read_text() == UNRELEASED_INITIAL_CONTENT + assert changelog.unreleased.read_text() == UNRELEASED_INITIAL_CONTENT @staticmethod - def test_create_versioned_changelog(changelogs, mock_dependencies): - changelogs._create_versioned_changes(SampleContent.changelog) - saved_text = changelogs.versioned_changelog_md.read_text() + def test_create_versioned_changes(changelog, mock_dependencies): + changelog._create_versioned_changes(SAMPLE.unreleased) + saved_text = changelog.versioned_changes.read_text() assert "1.0.0" in saved_text - assert SampleContent.changelog in saved_text - - @staticmethod - def test_extract_unreleased_notes(changelogs, unreleased_md): - actual = changelogs._extract_unreleased_notes() - expected = "## Summary\n" + SampleContent.changelog + "\n" - assert actual == expected + assert SAMPLE.unreleased_body in saved_text @staticmethod - def test_dependency_changes(changelogs, mock_dependencies): - result = changelogs._dependency_changes() - assert result == ( - "\n" - "### `main`\n\n" - "* Updated dependency `package1:0.0.1` to `0.1.0`\n" - "\n" - "### `dev`\n\n" - "* Added dependency `package2:0.2.0`\n" + def test_dependency_changes(changelog, mock_dependencies): + actual = changelog._dependency_changes() + expected = _markdown( + """ + ## Dependency Updates + ### `main` + * Updated dependency `package1:0.0.1` to `0.1.0` + ### `dev` + * Added dependency `package2:0.2.0` + """ ) + assert expected == actual @staticmethod def test_dependency_changes_without_latest_version( - changelogs, mock_new_dependencies + changelog, mock_new_dependencies ): - result = changelogs._dependency_changes() - assert result == ( - "\n" - "### `main`\n\n" - "* Added dependency `package1:0.1.0`\n" - "\n" - "### `dev`\n\n" - "* Added dependency `package2:0.2.0`\n" + actual = changelog._dependency_changes() + expected = _markdown( + """ + ## Dependency Updates + ### `main` + * Added dependency `package1:0.1.0` + ### `dev` + * Added dependency `package2:0.2.0` + """ ) + assert expected == actual @staticmethod @pytest.mark.parametrize( @@ -202,22 +210,22 @@ def test_dependency_changes_without_latest_version( ), ], ) - def test_sort_groups(changelogs, groups, expected): - result = changelogs._sort_groups(groups) + def test_sort_groups(changelog, groups, expected): + result = changelog._sort_groups(groups) assert result == expected @staticmethod - def test_update_table_of_contents(changelogs, changes_md): - changelogs._update_table_of_contents() + def test_update_table_of_contents(changelog, changes_md): + changelog._update_table_of_contents() - assert changelogs.changelog_md.read_text() == SampleContent.altered_changes + assert changelog.changelog.read_text() == SAMPLE.new_changelog @staticmethod - def test_prepare_release(changelogs, mock_dependencies, unreleased_md, changes_md): - changelogs.prepare_release() - assert changelogs.changelog_md.read_text() == SampleContent.altered_changes - assert changelogs.unreleased_md.read_text() == UNRELEASED_INITIAL_CONTENT - versioned = changelogs.versioned_changelog_md.read_text() + def test_prepare_release(changelog, mock_dependencies, unreleased_md, changes_md): + changelog.prepare_release() + assert changelog.changelog.read_text() == SAMPLE.new_changelog + assert changelog.unreleased.read_text() == UNRELEASED_INITIAL_CONTENT + versioned = Markdown.read(changelog.versioned_changes) assert versioned == expected_changes_file_content(with_dependencies=True) @staticmethod @@ -226,23 +234,23 @@ def test_update_latest( mock_no_dependencies, previous_dependencies, dependencies, - changelogs, + changelog, unreleased_md, changes_md, ): - changelogs.prepare_release() + changelog.prepare_release() mock_changelog(monkeypatch, previous_dependencies, dependencies) - changelogs.update_latest() - versioned = changelogs.versioned_changelog_md.read_text() + changelog.update_latest() + versioned = Markdown.read(changelog.versioned_changes) assert versioned == expected_changes_file_content(with_dependencies=True) @staticmethod def test_prepare_release_with_no_dependencies( - changelogs, mock_no_dependencies, unreleased_md, changes_md + changelog, mock_no_dependencies, unreleased_md, changes_md ): - changelogs.prepare_release() + changelog.prepare_release() - assert changelogs.changelog_md.read_text() == SampleContent.altered_changes - assert changelogs.unreleased_md.read_text() == UNRELEASED_INITIAL_CONTENT - versioned = changelogs.versioned_changelog_md.read_text() + assert changelog.changelog.read_text() == SAMPLE.new_changelog + assert changelog.unreleased.read_text() == UNRELEASED_INITIAL_CONTENT + versioned = Markdown.read(changelog.versioned_changes) assert versioned == expected_changes_file_content() From 27c7fb381383644dc0693df59019b30900bce008 Mon Sep 17 00:00:00 2001 From: ckunki Date: Thu, 9 Apr 2026 09:18:47 +0200 Subject: [PATCH 08/37] merged latest changes from markdown (3) --- exasol/toolbox/util/release/changelog.py | 4 +- exasol/toolbox/util/release/markdown.py | 80 +++++++++++------------- test/unit/util/release/changelog_test.py | 8 +-- test/unit/util/release/test_markdown.py | 28 +++++++-- 4 files changed, 65 insertions(+), 55 deletions(-) diff --git a/exasol/toolbox/util/release/changelog.py b/exasol/toolbox/util/release/changelog.py index 8f6dc3f27..fc7b258e4 100644 --- a/exasol/toolbox/util/release/changelog.py +++ b/exasol/toolbox/util/release/changelog.py @@ -133,10 +133,10 @@ def _create_versioned_changes(self, initial_content: str) -> None: unreleased_content: the content of the (not yet versioned) changes """ - versioned = Markdown.parse(initial_content) + versioned = Markdown.from_text(initial_content) versioned.title = f"# {self.version} - {datetime.today().strftime('%Y-%m-%d')}" if dependency_changes := self._dependency_changes(): - versioned.replace_child(dependency_changes) + versioned.replace_or_append_child(dependency_changes) # if resolved_vulnerabilities := self._resolved_vulnerabilities(): # if section := versioned.child(resolved_vulnerabilities.title): # section.intro = resolved_vulnerabilities diff --git a/exasol/toolbox/util/release/markdown.py b/exasol/toolbox/util/release/markdown.py index 0030ccc83..bbf08c5cf 100644 --- a/exasol/toolbox/util/release/markdown.py +++ b/exasol/toolbox/util/release/markdown.py @@ -1,34 +1,23 @@ """ -A project's Changelog is expected to list the changes coming with each of -the project's releases. The Changelog contains a file changelog.md with the -table of contents. and zero or more changes files. - -Each changes file is named changes_*.md and describes the changes for a -specific release. The * in the file name equals the version number of the -related release. - -All files are in Markdown syntax, divided into sections. Each section is -identified by its title which should be unique as is represented by class -Markdown. - -A section may consist of a prefix and a suffix, either might be empty. The -prefix are some introductory sentences, the suffix is the list of issues in -this section. Optionally each section can contain subsections as children. - -Method Markdown.replace_prefix() adds such a prefix or replaces it, when the -section already has one. - -The first line of each changes file must be the title describing the version -number, date and name of the release, followed by zero or multiple -sections. The first section is a summary, each other section lists the issues -(aka. tickets) of a particular category the are resolved by this -release. Categories are security, bugfixes, features, documentation, and -refactorings. +Class Markdown represents a file in markdown syntax with some additional +constraints: + +* The file must start with a title in the first line. +* Each subsequent title must be of a higher level, ie. start with more "#" + characters than the top-level title. + +Each title starts a section, optionally containing an additional intro and a +bullet list of items. + +Each section can also contain subsections as children, hence sections can be +nested up to the top-level section representing the whole file. """ from __future__ import annotations import io +from pathlib import Path +from typing import Optional class ParseError(Exception): @@ -110,14 +99,15 @@ def _check(self, child: Markdown) -> Markdown: ) return child - def add_child(self, child: Markdown, pos: int = 1) -> None: + def add_child(self, child: Markdown, pos: int = 1) -> Markdown: """ Insert the specified section as child at the specified position. """ self.children.insert(pos, self._check(child)) + return self - def replace_child(self, child: Markdown) -> Markdown: + def replace_or_append_child(self, child: Markdown) -> Markdown: """ If there is a child with the same title then replace this child otherwise append the specified child. @@ -142,12 +132,28 @@ def __eq__(self, other) -> bool: return isinstance(other, Markdown) and self.rendered == other.rendered @classmethod - def read(cls, path: Path) -> Markdown: - return cls.parse(path.read_text()) + def read(cls, file: Path) -> Markdown: + """ + Parse Markdown instance from the provided file. + """ + + with file.open("r") as stream: + return cls.parse(stream) @classmethod - def parse(cls, content: str) -> Markdown: - stream = io.StringIO(content) + def from_text(cls, text: str) -> Markdown: + """ + Parse Markdown instance from the provided text. + """ + + return cls.parse(io.StringIO(text)) + + @classmethod + def parse(cls, stream: io.TextIOBase) -> Markdown: + """ + Parse Markdown instance from the provided stream. + """ + line = stream.readline() if not is_title(line): raise ParseError( @@ -179,15 +185,3 @@ def _parse(cls, stream: io.TextIOBase, title: str) -> tuple[Markdown, str]: child, line = Markdown._parse(stream, title=line) children.append(child) return Markdown(title, intro.strip("\n"), items.strip("\n"), children), line - - -def sample(): - content = "" - changes = Markdown.parse(content) - resolved_vulnerabilities = "" - intro = resolved_vulnerabilities - title = "# title" - if section := changes.child(title): - section.intro = intro - else: - changes.add_child(Markdown(title, intro)) diff --git a/test/unit/util/release/changelog_test.py b/test/unit/util/release/changelog_test.py index 7b8bae7c5..322fb338b 100644 --- a/test/unit/util/release/changelog_test.py +++ b/test/unit/util/release/changelog_test.py @@ -18,7 +18,7 @@ class SampleData: def __init__(self, unreleased: str, old_changelog: str, new_changelog: str): - self.unreleased = Markdown.parse(unreleased).rendered + self.unreleased = Markdown.from_text(unreleased).rendered self.unreleased_body = "\n".join(self.unreleased.splitlines()[1:]) self.old_changelog = old_changelog self.new_changelog = new_changelog @@ -79,11 +79,11 @@ def __init__(self, unreleased: str, old_changelog: str, new_changelog: str): def _markdown(content: str) -> Markdown: - return Markdown.parse(cleandoc(content)) + return Markdown.from_text(cleandoc(content)) def expected_changes_file_content(with_dependencies: bool = False) -> Markdown: - changes = Markdown.parse(SAMPLE.unreleased) + changes = Markdown.from_text(SAMPLE.unreleased) changes.title = f"# 1.0.0 - {datetime.today().strftime('%Y-%m-%d')}" if not with_dependencies: return changes @@ -97,7 +97,7 @@ def expected_changes_file_content(with_dependencies: bool = False) -> Markdown: * Added dependency `package2:0.2.0` """ ) - return changes.replace_child(dependencies) + return changes.replace_or_append_child(dependencies) @pytest.fixture(scope="function") diff --git a/test/unit/util/release/test_markdown.py b/test/unit/util/release/test_markdown.py index 24013f111..83175951c 100644 --- a/test/unit/util/release/test_markdown.py +++ b/test/unit/util/release/test_markdown.py @@ -8,6 +8,8 @@ ParseError, ) +import pytest +import pytest class Scenario: def __init__( @@ -18,7 +20,7 @@ def __init__( self.expected_children = expected_children def create_testee(self) -> Markdown: - return Markdown.parse(self.initial) + return Markdown.from_text(self.initial) INVALID_MARKDOWN = cleandoc(""" @@ -137,7 +139,7 @@ def illegal_child() -> Markdown: def test_no_title_error(): with pytest.raises(ParseError, match="First line of markdown file must be a title"): - Markdown.parse("body\n# title") + Markdown.from_text("body\n# title") def test_additional_line_error(): @@ -145,7 +147,7 @@ def test_additional_line_error(): 'additional line "# Another Title" after top-level section "# Title".' ) with pytest.raises(ParseError, match=expected_error): - Markdown.parse(INVALID_MARKDOWN) + Markdown.from_text(INVALID_MARKDOWN) def test_constructor_illegal_child(illegal_child: Markdown): @@ -153,6 +155,20 @@ def test_constructor_illegal_child(illegal_child: Markdown): Markdown("# title", children=[illegal_child]) +def test_equals() -> None: + testee = MINIMAL.create_testee() + other = MINIMAL.create_testee() + assert other == testee + other.title = "# other" + assert other != testee + + +def test_test_read(tmp_path) -> None: + file = tmp_path / "sample.md" + file.write_text(MINIMAL.initial) + assert Markdown.read(file) == MINIMAL.create_testee() + + ALL_SCENARIOS = [MINIMAL, WITH_CHILD, TWO_CHILDREN, NESTED] @@ -193,7 +209,7 @@ def test_add_child(sample_child: Markdown, scenario: Scenario, pos: int): def test_replace_illegal_child(illegal_child): testee = WITH_CHILD.create_testee() with pytest.raises(IllegalChild): - testee.replace_child(illegal_child) + testee.replace_or_append_child(illegal_child) @pytest.mark.parametrize("scenario", ALL_SCENARIOS) @@ -203,7 +219,7 @@ def test_replace_existing_child(scenario: Scenario): old_rendered = testee.rendered new_child = Markdown(old_child.title, "new intro") expected = old_rendered.replace(old_child.rendered, new_child.rendered) - testee.replace_child(new_child) + testee.replace_or_append_child(new_child) assert testee.rendered == expected @@ -211,7 +227,7 @@ def test_replace_existing_child(scenario: Scenario): def test_replace_non_existing_child(scenario: Scenario, sample_child: Markdown): testee = scenario.create_testee() expected = len(testee.children) + 1 - testee.replace_child(sample_child) + testee.replace_or_append_child(sample_child) assert len(testee.children) == expected assert testee.children[-1] == sample_child From 1527212ffb082d651d7e700d21d170c2f08c0ae1 Mon Sep 17 00:00:00 2001 From: ckunki Date: Thu, 9 Apr 2026 10:31:02 +0200 Subject: [PATCH 09/37] Activated reporting vulnerablities Fixed tests --- exasol/toolbox/util/release/changelog.py | 19 +++++--- test/unit/util/release/changelog_test.py | 58 +++++++++++++++++------- 2 files changed, 53 insertions(+), 24 deletions(-) diff --git a/exasol/toolbox/util/release/changelog.py b/exasol/toolbox/util/release/changelog.py index fc7b258e4..b4dd92c76 100644 --- a/exasol/toolbox/util/release/changelog.py +++ b/exasol/toolbox/util/release/changelog.py @@ -6,12 +6,17 @@ from inspect import cleandoc from pathlib import Path +from exasol.toolbox.util.dependencies.audit import ( + get_vulnerabilities, + get_vulnerabilities_from_latest_tag, +) from exasol.toolbox.util.dependencies.poetry_dependencies import ( get_dependencies, get_dependencies_from_latest_tag, ) from exasol.toolbox.util.dependencies.shared_models import LatestTagNotFoundError from exasol.toolbox.util.dependencies.track_changes import DependencyChanges +from exasol.toolbox.util.dependencies.track_vulnerabilities import DependenciesAudit from exasol.toolbox.util.release.markdown import Markdown from exasol.toolbox.util.version import Version @@ -120,8 +125,8 @@ def get_changed_files(self) -> list[Path]: def _resolved_vulnerabilities(self) -> Markdown | None: report = DependenciesAudit( - previous_vulnerabilities=get_vulnerabilities_from_latest_tag(path), - current_vulnerabilities=get_vulnerabilities(path), + previous_vulnerabilities=get_vulnerabilities_from_latest_tag(self.root_path), + current_vulnerabilities=get_vulnerabilities(self.root_path), ).report_resolved_vulnerabilities() return Markdown("## Security Issues", report) if report else None @@ -137,11 +142,11 @@ def _create_versioned_changes(self, initial_content: str) -> None: versioned.title = f"# {self.version} - {datetime.today().strftime('%Y-%m-%d')}" if dependency_changes := self._dependency_changes(): versioned.replace_or_append_child(dependency_changes) - # if resolved_vulnerabilities := self._resolved_vulnerabilities(): - # if section := versioned.child(resolved_vulnerabilities.title): - # section.intro = resolved_vulnerabilities - # else: - # versioned.add_child(resolved_vulnerabilities) + if resolved_vulnerabilities := self._resolved_vulnerabilities(): + if section := versioned.child(resolved_vulnerabilities.title): + section.intro = resolved_vulnerabilities + else: + versioned.add_child(resolved_vulnerabilities) self.versioned_changes.write_text(versioned.rendered) def prepare_release(self) -> Changelog: diff --git a/test/unit/util/release/changelog_test.py b/test/unit/util/release/changelog_test.py index 322fb338b..9cc3248f9 100644 --- a/test/unit/util/release/changelog_test.py +++ b/test/unit/util/release/changelog_test.py @@ -15,6 +15,7 @@ from exasol.toolbox.util.release.markdown import Markdown from exasol.toolbox.util.version import Version +import pytest class SampleData: def __init__(self, unreleased: str, old_changelog: str, new_changelog: str): @@ -121,31 +122,55 @@ def unreleased_md(changelog): changelog.unreleased.write_text(SAMPLE.unreleased) -def mock_changelog(monkeypatch, old_dependencies, new_dependencies): - for func, value in ( - ("get_dependencies_from_latest_tag", old_dependencies), - ("get_dependencies", new_dependencies), +DependencyChanges = tuple[Mock | dict, Mock | dict] | None +VulnerabilityChanges = tuple[Mock | dict, Mock | dict] | None + + +@pytest.fixture +def mock_changelog(monkeypatch) -> Callable[[DependencyChanges, VulnerabilityChanges], None]: + """ + Enable simulating pecific changes in dependencies or vulnerabilities + between the latest tag (last release) and the current release. + """ + def mock( + dependencies: DependencyChanges = None, + vulnerabilities: VulnerabilityChanges = None, ): - mock = value if isinstance(value, Mock) else Mock(return_value=value) - monkeypatch.setattr(impl, func, mock) + dependencies = dependencies or ({}, {}) + vulnerabilities = vulnerabilities or ([], []) + for func, value in ( + ("get_dependencies_from_latest_tag", dependencies[0]), + ("get_dependencies", dependencies[1]), + ("get_vulnerabilities_from_latest_tag", vulnerabilities[0]), + ("get_vulnerabilities", vulnerabilities[1]), + ): + mock = value if isinstance(value, Mock) else Mock(return_value=value) + monkeypatch.setattr(impl, func, mock) + + return mock @pytest.fixture(scope="function") -def mock_dependencies(monkeypatch, previous_dependencies, dependencies): - mock_changelog(monkeypatch, previous_dependencies, dependencies) +def mock_no_dependencies(mock_changelog): + mock_changelog() @pytest.fixture(scope="function") -def mock_new_dependencies(monkeypatch, dependencies): - mock_changelog(monkeypatch, Mock(side_effect=LatestTagNotFoundError), dependencies) +def mock_dependencies(mock_changelog, previous_dependencies, dependencies): + mock_changelog(dependencies=(previous_dependencies, dependencies)) @pytest.fixture(scope="function") -def mock_no_dependencies(monkeypatch): - mock_changelog(monkeypatch, {}, {}) +def mock_new_dependencies(mock_changelog, dependencies): + mock_changelog(dependencies=(Mock(side_effect=LatestTagNotFoundError), dependencies)) + +@pytest.fixture(scope="function") +def mock_no_vulnerabilities(mock_changelog): + mock_changelog() -class TestChangelogs: + +class TestChangelog: """ As some methods in the class `Changelog` modify files, it is required that the fixtures which create the sample files (changelog.md, @@ -217,7 +242,6 @@ def test_sort_groups(changelog, groups, expected): @staticmethod def test_update_table_of_contents(changelog, changes_md): changelog._update_table_of_contents() - assert changelog.changelog.read_text() == SAMPLE.new_changelog @staticmethod @@ -230,16 +254,16 @@ def test_prepare_release(changelog, mock_dependencies, unreleased_md, changes_md @staticmethod def test_update_latest( - monkeypatch, - mock_no_dependencies, + mock_changelog, previous_dependencies, dependencies, changelog, unreleased_md, changes_md, ): + mock_changelog() changelog.prepare_release() - mock_changelog(monkeypatch, previous_dependencies, dependencies) + mock_changelog(dependencies=(previous_dependencies, dependencies)) changelog.update_latest() versioned = Markdown.read(changelog.versioned_changes) assert versioned == expected_changes_file_content(with_dependencies=True) From 4755e20ec15b53c3c3db4b9e7d722cdc59674a24 Mon Sep 17 00:00:00 2001 From: ckunki Date: Thu, 9 Apr 2026 10:39:56 +0200 Subject: [PATCH 10/37] Simplified tests --- test/unit/util/release/changelog_test.py | 44 ++++++++++-------------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/test/unit/util/release/changelog_test.py b/test/unit/util/release/changelog_test.py index 9cc3248f9..a61ac602a 100644 --- a/test/unit/util/release/changelog_test.py +++ b/test/unit/util/release/changelog_test.py @@ -17,16 +17,12 @@ import pytest -class SampleData: - def __init__(self, unreleased: str, old_changelog: str, new_changelog: str): - self.unreleased = Markdown.from_text(unreleased).rendered - self.unreleased_body = "\n".join(self.unreleased.splitlines()[1:]) - self.old_changelog = old_changelog - self.new_changelog = new_changelog +def _markdown(content: str) -> Markdown: + return Markdown.from_text(cleandoc(content)) -SAMPLE = SampleData( - unreleased=cleandoc( +class SampleContent: + unreleased = _markdown( """ # Unreleased ## Summary @@ -41,8 +37,8 @@ def __init__(self, unreleased: str, old_changelog: str, new_changelog: str): ## Refactorings * Some refactoring """ - ), - old_changelog=cleandoc( + ).rendered + old_changelog = cleandoc( """ # Changelog @@ -57,8 +53,8 @@ def __init__(self, unreleased: str, old_changelog: str, new_changelog: str): changes_0.1.0 ``` """ - ), - new_changelog=cleandoc( + ) + new_changelog = cleandoc( """ # Changelog @@ -75,16 +71,11 @@ def __init__(self, unreleased: str, old_changelog: str, new_changelog: str): changes_0.1.0 ``` """ - ), -) - - -def _markdown(content: str) -> Markdown: - return Markdown.from_text(cleandoc(content)) + ) def expected_changes_file_content(with_dependencies: bool = False) -> Markdown: - changes = Markdown.from_text(SAMPLE.unreleased) + changes = Markdown.from_text(SampleContent.unreleased) changes.title = f"# 1.0.0 - {datetime.today().strftime('%Y-%m-%d')}" if not with_dependencies: return changes @@ -114,12 +105,12 @@ def changelog(tmp_path) -> Changelog: @pytest.fixture(scope="function") def changes_md(changelog): - changelog.changelog.write_text(SAMPLE.old_changelog) + changelog.changelog.write_text(SampleContent.old_changelog) @pytest.fixture(scope="function") def unreleased_md(changelog): - changelog.unreleased.write_text(SAMPLE.unreleased) + changelog.unreleased.write_text(SampleContent.unreleased) DependencyChanges = tuple[Mock | dict, Mock | dict] | None @@ -187,11 +178,12 @@ def test_create_new_unreleased(changelog): @staticmethod def test_create_versioned_changes(changelog, mock_dependencies): - changelog._create_versioned_changes(SAMPLE.unreleased) + changelog._create_versioned_changes(SampleContent.unreleased) saved_text = changelog.versioned_changes.read_text() assert "1.0.0" in saved_text - assert SAMPLE.unreleased_body in saved_text + unreleased_body = "\n".join(SampleContent.unreleased.splitlines()[1:]) + assert unreleased_body in saved_text @staticmethod def test_dependency_changes(changelog, mock_dependencies): @@ -242,12 +234,12 @@ def test_sort_groups(changelog, groups, expected): @staticmethod def test_update_table_of_contents(changelog, changes_md): changelog._update_table_of_contents() - assert changelog.changelog.read_text() == SAMPLE.new_changelog + assert changelog.changelog.read_text() == SampleContent.new_changelog @staticmethod def test_prepare_release(changelog, mock_dependencies, unreleased_md, changes_md): changelog.prepare_release() - assert changelog.changelog.read_text() == SAMPLE.new_changelog + assert changelog.changelog.read_text() == SampleContent.new_changelog assert changelog.unreleased.read_text() == UNRELEASED_INITIAL_CONTENT versioned = Markdown.read(changelog.versioned_changes) assert versioned == expected_changes_file_content(with_dependencies=True) @@ -274,7 +266,7 @@ def test_prepare_release_with_no_dependencies( ): changelog.prepare_release() - assert changelog.changelog.read_text() == SAMPLE.new_changelog + assert changelog.changelog.read_text() == SampleContent.new_changelog assert changelog.unreleased.read_text() == UNRELEASED_INITIAL_CONTENT versioned = Markdown.read(changelog.versioned_changes) assert versioned == expected_changes_file_content() From 5867a7f9f749663f832aa44a379f199b18f797b3 Mon Sep 17 00:00:00 2001 From: ckunki Date: Thu, 9 Apr 2026 11:05:34 +0200 Subject: [PATCH 11/37] Added tests for changelog incl. vulnerabilities --- exasol/toolbox/util/release/markdown.py | 3 + test/unit/util/release/changelog_test.py | 129 +++++++++++++++-------- 2 files changed, 89 insertions(+), 43 deletions(-) diff --git a/exasol/toolbox/util/release/markdown.py b/exasol/toolbox/util/release/markdown.py index bbf08c5cf..406c73d34 100644 --- a/exasol/toolbox/util/release/markdown.py +++ b/exasol/toolbox/util/release/markdown.py @@ -131,6 +131,9 @@ def elements(): def __eq__(self, other) -> bool: return isinstance(other, Markdown) and self.rendered == other.rendered + def __str__(self) -> str: + return self.rendered + @classmethod def read(cls, file: Path) -> Markdown: """ diff --git a/test/unit/util/release/changelog_test.py b/test/unit/util/release/changelog_test.py index a61ac602a..3ed93b058 100644 --- a/test/unit/util/release/changelog_test.py +++ b/test/unit/util/release/changelog_test.py @@ -1,5 +1,6 @@ from __future__ import annotations +from collections.abc import Callable from datetime import datetime from inspect import cleandoc from unittest.mock import Mock @@ -15,15 +16,13 @@ from exasol.toolbox.util.release.markdown import Markdown from exasol.toolbox.util.version import Version -import pytest def _markdown(content: str) -> Markdown: return Markdown.from_text(cleandoc(content)) class SampleContent: - unreleased = _markdown( - """ + unreleased = _markdown(""" # Unreleased ## Summary Summary of changes. @@ -36,10 +35,8 @@ class SampleContent: ## Refactorings * Some refactoring - """ - ).rendered - old_changelog = cleandoc( - """ + """).rendered + old_changelog = cleandoc(""" # Changelog * [unreleased](unreleased.md) @@ -52,10 +49,8 @@ class SampleContent: unreleased changes_0.1.0 ``` - """ - ) - new_changelog = cleandoc( - """ + """) + new_changelog = cleandoc(""" # Changelog * [unreleased](unreleased.md) @@ -70,26 +65,36 @@ class SampleContent: changes_1.0.0 changes_0.1.0 ``` - """ - ) + """) -def expected_changes_file_content(with_dependencies: bool = False) -> Markdown: - changes = Markdown.from_text(SampleContent.unreleased) - changes.title = f"# 1.0.0 - {datetime.today().strftime('%Y-%m-%d')}" - if not with_dependencies: - return changes - - dependencies = _markdown( - f""" +def expected_changes_file_content( + with_dependencies: bool = False, + with_vulnerabilities: bool = False, +) -> Markdown: + dependencies = _markdown(f""" ## Dependency Updates ### `main` * Updated dependency `package1:0.0.1` to `0.1.0` ### `dev` * Added dependency `package2:0.2.0` - """ - ) - return changes.replace_or_append_child(dependencies) + """) + vulnerabilities = _markdown(""" + ## Security Issues + + This release fixes vulnerabilities by updating dependencies: + + | Dependency | Vulnerability | Affected | Fixed in | + |------------|---------------|----------|----------| + | jinja2 | CVE-2025-27516 | 3.1.5 | 3.1.6 | + """) + changes = Markdown.from_text(SampleContent.unreleased) + changes.title = f"# 1.0.0 - {datetime.today().strftime('%Y-%m-%d')}" + if with_vulnerabilities: + changes.add_child(vulnerabilities) + if with_dependencies: + changes.replace_or_append_child(dependencies) + return changes @pytest.fixture(scope="function") @@ -117,12 +122,20 @@ def unreleased_md(changelog): VulnerabilityChanges = tuple[Mock | dict, Mock | dict] | None +@pytest.fixture(scope="function") +def dependency_changes(previous_dependencies, dependencies) -> DependencyChanges: + return (previous_dependencies, dependencies) + + @pytest.fixture -def mock_changelog(monkeypatch) -> Callable[[DependencyChanges, VulnerabilityChanges], None]: +def mock_changelog( + monkeypatch, +) -> Callable[[DependencyChanges, VulnerabilityChanges], None]: """ Enable simulating pecific changes in dependencies or vulnerabilities between the latest tag (last release) and the current release. """ + def mock( dependencies: DependencyChanges = None, vulnerabilities: VulnerabilityChanges = None, @@ -147,18 +160,33 @@ def mock_no_dependencies(mock_changelog): @pytest.fixture(scope="function") -def mock_dependencies(mock_changelog, previous_dependencies, dependencies): - mock_changelog(dependencies=(previous_dependencies, dependencies)) +def mock_dependencies(mock_changelog, dependency_changes): + mock_changelog(dependencies=dependency_changes) @pytest.fixture(scope="function") -def mock_new_dependencies(mock_changelog, dependencies): - mock_changelog(dependencies=(Mock(side_effect=LatestTagNotFoundError), dependencies)) +def vulnerability_changes(sample_vulnerability) -> VulnerabilityChanges: + """ + Simulate resolved vulnerabilities. + """ + + previous = [sample_vulnerability.vulnerability] + current = [] + return (previous, current) @pytest.fixture(scope="function") -def mock_no_vulnerabilities(mock_changelog): - mock_changelog() +def mock_dependencies_and_vulnerabiltiies( + mock_changelog, dependency_changes, vulnerability_changes +): + mock_changelog(dependency_changes, vulnerability_changes) + + +@pytest.fixture(scope="function") +def mock_new_dependencies(mock_changelog, dependencies): + mock_changelog( + dependencies=(Mock(side_effect=LatestTagNotFoundError), dependencies) + ) class TestChangelog: @@ -188,15 +216,13 @@ def test_create_versioned_changes(changelog, mock_dependencies): @staticmethod def test_dependency_changes(changelog, mock_dependencies): actual = changelog._dependency_changes() - expected = _markdown( - """ + expected = _markdown(""" ## Dependency Updates ### `main` * Updated dependency `package1:0.0.1` to `0.1.0` ### `dev` * Added dependency `package2:0.2.0` - """ - ) + """) assert expected == actual @staticmethod @@ -204,15 +230,13 @@ def test_dependency_changes_without_latest_version( changelog, mock_new_dependencies ): actual = changelog._dependency_changes() - expected = _markdown( - """ + expected = _markdown(""" ## Dependency Updates ### `main` * Added dependency `package1:0.1.0` ### `dev` * Added dependency `package2:0.2.0` - """ - ) + """) assert expected == actual @staticmethod @@ -244,21 +268,40 @@ def test_prepare_release(changelog, mock_dependencies, unreleased_md, changes_md versioned = Markdown.read(changelog.versioned_changes) assert versioned == expected_changes_file_content(with_dependencies=True) + @staticmethod + def test_2prepare_release( + changelog, + mock_dependencies_and_vulnerabiltiies, + unreleased_md, + changes_md, + ): + changelog.prepare_release() + assert changelog.changelog.read_text() == SampleContent.new_changelog + assert changelog.unreleased.read_text() == UNRELEASED_INITIAL_CONTENT + versioned = Markdown.read(changelog.versioned_changes) + expected = expected_changes_file_content( + with_dependencies=True, with_vulnerabilities=True + ) + assert versioned == expected + @staticmethod def test_update_latest( mock_changelog, - previous_dependencies, - dependencies, + dependency_changes, + vulnerability_changes, changelog, unreleased_md, changes_md, ): mock_changelog() changelog.prepare_release() - mock_changelog(dependencies=(previous_dependencies, dependencies)) + mock_changelog(dependency_changes, vulnerability_changes) changelog.update_latest() versioned = Markdown.read(changelog.versioned_changes) - assert versioned == expected_changes_file_content(with_dependencies=True) + expected = expected_changes_file_content( + with_dependencies=True, with_vulnerabilities=True, + ) + assert versioned == expected @staticmethod def test_prepare_release_with_no_dependencies( From 65212ffb22037d057f53f029912d8f2c8104bec2 Mon Sep 17 00:00:00 2001 From: ckunki Date: Thu, 9 Apr 2026 13:30:59 +0200 Subject: [PATCH 12/37] Refactored tests added test for inserted vulnerabilities --- exasol/toolbox/util/release/changelog.py | 2 +- exasol/toolbox/util/release/markdown.py | 4 +- test/unit/util/release/changelog_test.py | 115 +++++++++++++++-------- test/unit/util/release/test_markdown.py | 10 ++ 4 files changed, 87 insertions(+), 44 deletions(-) diff --git a/exasol/toolbox/util/release/changelog.py b/exasol/toolbox/util/release/changelog.py index b4dd92c76..8db96f8fa 100644 --- a/exasol/toolbox/util/release/changelog.py +++ b/exasol/toolbox/util/release/changelog.py @@ -144,7 +144,7 @@ def _create_versioned_changes(self, initial_content: str) -> None: versioned.replace_or_append_child(dependency_changes) if resolved_vulnerabilities := self._resolved_vulnerabilities(): if section := versioned.child(resolved_vulnerabilities.title): - section.intro = resolved_vulnerabilities + section.intro = resolved_vulnerabilities.intro else: versioned.add_child(resolved_vulnerabilities) self.versioned_changes.write_text(versioned.rendered) diff --git a/exasol/toolbox/util/release/markdown.py b/exasol/toolbox/util/release/markdown.py index 406c73d34..68d40248b 100644 --- a/exasol/toolbox/util/release/markdown.py +++ b/exasol/toolbox/util/release/markdown.py @@ -38,7 +38,7 @@ def is_title(line: str) -> bool: def is_list_item(line: str) -> bool: - return line and (line.startswith("#") or line.startswith("-")) + return line and (line.startswith("*") or line.startswith("-")) def is_intro(line: str) -> bool: @@ -181,7 +181,7 @@ def _parse(cls, stream: io.TextIOBase, title: str) -> tuple[Markdown, str]: intro += line line = stream.readline() if is_list_item(line): - while not is_title(line): + while line and not is_title(line): items += line line = stream.readline() while is_title(line) and level(title) < level(line): diff --git a/test/unit/util/release/changelog_test.py b/test/unit/util/release/changelog_test.py index 3ed93b058..29b547044 100644 --- a/test/unit/util/release/changelog_test.py +++ b/test/unit/util/release/changelog_test.py @@ -2,6 +2,10 @@ from collections.abc import Callable from datetime import datetime +from enum import ( + Flag, + auto, +) from inspect import cleandoc from unittest.mock import Mock @@ -68,31 +72,41 @@ class SampleContent: """) -def expected_changes_file_content( - with_dependencies: bool = False, - with_vulnerabilities: bool = False, +class Expect(Flag): + NONE = auto() + VULNERABILITIES = auto() + VULNERABILITIES_INSERTED = auto() + DEPENDENCIES = auto() + + +def expected_changes( + unreleased: str = SampleContent.unreleased, + include: Expect = Expect.NONE, ) -> Markdown: dependencies = _markdown(f""" - ## Dependency Updates - ### `main` - * Updated dependency `package1:0.0.1` to `0.1.0` - ### `dev` - * Added dependency `package2:0.2.0` - """) + ## Dependency Updates + ### `main` + * Updated dependency `package1:0.0.1` to `0.1.0` + ### `dev` + * Added dependency `package2:0.2.0` + """) + vulnerabilities = _markdown(""" - ## Security Issues + ## Security Issues - This release fixes vulnerabilities by updating dependencies: + This release fixes vulnerabilities by updating dependencies: - | Dependency | Vulnerability | Affected | Fixed in | - |------------|---------------|----------|----------| - | jinja2 | CVE-2025-27516 | 3.1.5 | 3.1.6 | - """) - changes = Markdown.from_text(SampleContent.unreleased) + | Dependency | Vulnerability | Affected | Fixed in | + |------------|---------------|----------|----------| + | jinja2 | CVE-2025-27516 | 3.1.5 | 3.1.6 | + """) + changes = Markdown.from_text(unreleased) changes.title = f"# 1.0.0 - {datetime.today().strftime('%Y-%m-%d')}" - if with_vulnerabilities: + if Expect.VULNERABILITIES in include: changes.add_child(vulnerabilities) - if with_dependencies: + if Expect.VULNERABILITIES_INSERTED in include: + changes.child("## Security Issues").intro = (vulnerabilities.intro) + if Expect.DEPENDENCIES in include: changes.replace_or_append_child(dependencies) return changes @@ -109,7 +123,7 @@ def changelog(tmp_path) -> Changelog: @pytest.fixture(scope="function") -def changes_md(changelog): +def changelog_md(changelog): changelog.changelog.write_text(SampleContent.old_changelog) @@ -118,6 +132,17 @@ def unreleased_md(changelog): changelog.unreleased.write_text(SampleContent.unreleased) +@pytest.fixture(scope="function") +def unreleased_with_security_issues(changelog) -> str: + security = _markdown(""" + ## Security Issues + * #123: Fixed vulnerability + """) + md = _markdown(SampleContent.unreleased).add_child(security) + changelog.unreleased.write_text(md.rendered) + return md.rendered + + DependencyChanges = tuple[Mock | dict, Mock | dict] | None VulnerabilityChanges = tuple[Mock | dict, Mock | dict] | None @@ -176,7 +201,12 @@ def vulnerability_changes(sample_vulnerability) -> VulnerabilityChanges: @pytest.fixture(scope="function") -def mock_dependencies_and_vulnerabiltiies( +def mock_vulnerabilties(mock_changelog, vulnerability_changes): + mock_changelog(vulnerabilities=vulnerability_changes) + + +@pytest.fixture(scope="function") +def mock_dependencies_and_vulnerabilties( mock_changelog, dependency_changes, vulnerability_changes ): mock_changelog(dependency_changes, vulnerability_changes) @@ -256,32 +286,22 @@ def test_sort_groups(changelog, groups, expected): assert result == expected @staticmethod - def test_update_table_of_contents(changelog, changes_md): + def test_update_table_of_contents(changelog, changelog_md): changelog._update_table_of_contents() assert changelog.changelog.read_text() == SampleContent.new_changelog @staticmethod - def test_prepare_release(changelog, mock_dependencies, unreleased_md, changes_md): - changelog.prepare_release() - assert changelog.changelog.read_text() == SampleContent.new_changelog - assert changelog.unreleased.read_text() == UNRELEASED_INITIAL_CONTENT - versioned = Markdown.read(changelog.versioned_changes) - assert versioned == expected_changes_file_content(with_dependencies=True) - - @staticmethod - def test_2prepare_release( + def test_prepare_release( changelog, - mock_dependencies_and_vulnerabiltiies, + mock_dependencies_and_vulnerabilties, unreleased_md, - changes_md, + changelog_md, ): changelog.prepare_release() assert changelog.changelog.read_text() == SampleContent.new_changelog assert changelog.unreleased.read_text() == UNRELEASED_INITIAL_CONTENT versioned = Markdown.read(changelog.versioned_changes) - expected = expected_changes_file_content( - with_dependencies=True, with_vulnerabilities=True - ) + expected = expected_changes(include=Expect.DEPENDENCIES | Expect.VULNERABILITIES) assert versioned == expected @staticmethod @@ -291,25 +311,38 @@ def test_update_latest( vulnerability_changes, changelog, unreleased_md, - changes_md, + changelog_md, ): mock_changelog() changelog.prepare_release() mock_changelog(dependency_changes, vulnerability_changes) changelog.update_latest() versioned = Markdown.read(changelog.versioned_changes) - expected = expected_changes_file_content( - with_dependencies=True, with_vulnerabilities=True, - ) + expected = expected_changes(include=Expect.DEPENDENCIES | Expect.VULNERABILITIES) assert versioned == expected @staticmethod def test_prepare_release_with_no_dependencies( - changelog, mock_no_dependencies, unreleased_md, changes_md + changelog, mock_no_dependencies, unreleased_md, changelog_md ): changelog.prepare_release() assert changelog.changelog.read_text() == SampleContent.new_changelog assert changelog.unreleased.read_text() == UNRELEASED_INITIAL_CONTENT versioned = Markdown.read(changelog.versioned_changes) - assert versioned == expected_changes_file_content() + assert versioned == expected_changes() + + @staticmethod + def test_prepare_release_with_security_issues( + changelog, + mock_vulnerabilties, + unreleased_with_security_issues, + changelog_md, + ): + changelog.prepare_release() + versioned = Markdown.read(changelog.versioned_changes) + expected = expected_changes( + unreleased_with_security_issues, + include=Expect.VULNERABILITIES_INSERTED, + ) + assert versioned == expected diff --git a/test/unit/util/release/test_markdown.py b/test/unit/util/release/test_markdown.py index 83175951c..8c0c624cb 100644 --- a/test/unit/util/release/test_markdown.py +++ b/test/unit/util/release/test_markdown.py @@ -8,6 +8,7 @@ ParseError, ) +import pytest import pytest import pytest @@ -155,6 +156,15 @@ def test_constructor_illegal_child(illegal_child: Markdown): Markdown("# title", children=[illegal_child]) +def test_no_intro() -> None: + content = """ + # title + * #123: Fixed vulnerability + """ + testee = Markdown.from_text(cleandoc(content)) + assert testee == Markdown("# title", "", "* #123: Fixed vulnerability") + + def test_equals() -> None: testee = MINIMAL.create_testee() other = MINIMAL.create_testee() From 8033dd22774ed366a6a053ab21732a36883d23c4 Mon Sep 17 00:00:00 2001 From: ckunki Date: Thu, 9 Apr 2026 13:33:20 +0200 Subject: [PATCH 13/37] merged latest changes from markdown (4) --- test/unit/util/release/test_markdown.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/unit/util/release/test_markdown.py b/test/unit/util/release/test_markdown.py index 8c0c624cb..a6916439e 100644 --- a/test/unit/util/release/test_markdown.py +++ b/test/unit/util/release/test_markdown.py @@ -8,9 +8,6 @@ ParseError, ) -import pytest -import pytest -import pytest class Scenario: def __init__( From 9a1d02f29276618037a5806b5104f0602fd3bcae Mon Sep 17 00:00:00 2001 From: ckunki Date: Thu, 9 Apr 2026 16:45:34 +0200 Subject: [PATCH 14/37] Updated user guide --- .../features/creating_a_release.rst | 22 ++++++++------- .../features/managing_dependencies.rst | 27 ++++++++++-------- doc/user_guide/user_guide.rst | 28 ++++++++++--------- exasol/toolbox/nox/_dependencies.py | 3 +- 4 files changed, 44 insertions(+), 36 deletions(-) diff --git a/doc/user_guide/features/creating_a_release.rst b/doc/user_guide/features/creating_a_release.rst index a55a2a7a5..12f2a64e4 100644 --- a/doc/user_guide/features/creating_a_release.rst +++ b/doc/user_guide/features/creating_a_release.rst @@ -17,15 +17,17 @@ Preparing a Release * Updates the version in the ``pyproject.toml`` and ``version.py`` * Moves the content of unreleased changes file ``unreleased.md`` to a versioned changes file ``changes_.md`` - * Adds a description of dependency changes to the versioned changes file: + * Describes additional changes in the versioned changes file by comparing + file ``poetry.lock`` to the latest Git tag: - * Only direct dependencies are described, no transitive dependencies - * Changes are detected by comparing the current content of file - ``poetry.lock`` to the latest Git tag. - * Updates the ``changelog.md`` list with the newly created versioned changes file + * Resolved vulnerabilities based on `Pip Audit`_. + * Updated direct dependencies, excluding transitive dependencies + * Updates file ``changelog.md`` to list the newly created versioned changes file * Commits the changes (can be skipped with ``--no-add``) * Pushes the changes and creates a PR (can be skipped with ``--no-pr``) +.. _Pip Audit: https://pypi.org/project/pip-audit/ + #. Merge your **Pull Request** to the **default branch** #. Trigger the release @@ -53,12 +55,12 @@ Preparing a Release Your ``PROJECT_CONFIG`` needs to have the flag ``create_major_version_tags=True``. -Updating Dependencies After Having Prepared the Release -+++++++++++++++++++++++++++++++++++++++++++++++++++++++ +Updating The Versioned Changes File ++++++++++++++++++++++++++++++++++++ -If you need to update some more dependencies after running the nox session -``release:prepare`` you can update them in the changelog by running the nox -session ``release:update``. +If you need to update some dependencies after running the nox session +``release:prepare`` you can update the versioned changes file by running the +nox session ``release:update``. What to do if the Release Failed? diff --git a/doc/user_guide/features/managing_dependencies.rst b/doc/user_guide/features/managing_dependencies.rst index 4c636e170..a89b52735 100644 --- a/doc/user_guide/features/managing_dependencies.rst +++ b/doc/user_guide/features/managing_dependencies.rst @@ -1,12 +1,17 @@ -Managing dependencies -===================== +Managing Dependencies and Vulnerabilities +========================================= -+--------------------------+------------------+----------------------------------------+ -| Nox session | CI Usage | Action | -+==========================+==================+========================================+ -| ``dependency:licenses`` | ``report.yml`` | Uses ``pip-licenses`` to return | -| | | packages with their licenses | -+--------------------------+------------------+----------------------------------------+ -| ``dependency:audit`` | No | Uses ``pip-audit`` to return active | -| | | vulnerabilities in our dependencies | -+--------------------------+------------------+----------------------------------------+ ++------------------------------+----------------+-------------------------------------+ +| Nox session | CI Usage | Action | ++==============================+================+=====================================+ +| ``dependency:licenses`` | ``report.yml`` | Uses ``pip-licenses`` to return | +| | | packages with their licenses | ++------------------------------+----------------+-------------------------------------+ +| ``dependency:audit`` | No | Uses ``pip-audit`` to report active | +| | | vulnerabilities in our dependencies | ++------------------------------+----------------+-------------------------------------+ +| ``vulnerabilities:resolved`` | No | Uses ``pip-audit`` to report known | +| | | vulnerabilities in depdendencies | +| | | that have been resolved in | +| | | comparison to the last release. | ++------------------------------+----------------+-------------------------------------+ diff --git a/doc/user_guide/user_guide.rst b/doc/user_guide/user_guide.rst index 6106e019c..44f8af4d6 100644 --- a/doc/user_guide/user_guide.rst +++ b/doc/user_guide/user_guide.rst @@ -21,28 +21,30 @@ PTB simplifies keeping all of your projects up-to-date, secure, without bugs, us The PTB gains its name from employing a series of well-established tools to satisfy these goals: -* `Poetry`_ for packaging and managing dependencies -* `Nox`_ for using the tools via a common CLI +* `Bandit`_ for detecting security vulnerabilities * `Black`_ and `Ruff`_ for source code formatting -* `Pylint`_ / `Ruff` for linting * `Cookiecutter`_ for setting up new projects from a uniform template -* `Mypy`_ for static type checking * `Coverage`_ for measuring code coverage by tests -* `Bandit`_ for detecting security vulnerabilities -* `Sphinx`_ for generating the documentation +* `Mypy`_ for static type checking +* `Nox`_ for using the tools via a common CLI +* `Pip Audit`_ for known vulnerabilities in dependencies +* `Poetry`_ for packaging and managing dependencies +* `Pylint`_ / `Ruff` for linting * `Sonar`_ for reporting code quality based on the findings by other tools +* `Sphinx`_ for generating the documentation In rare cases you may need to disable a particular finding reported by one of these tools, see :ref:`ptb_troubleshooting`. -.. _Poetry: https://python-poetry.org -.. _Nox: https://nox.thea.codes/en/stable/ +.. _Bandit: https://bandit.readthedocs.io/en/latest/ .. _Black: https://black.readthedocs.io/en/stable/ -.. _Ruff: https://docs.astral.sh/ruff -.. _Pylint: https://pylint.readthedocs.io/en/stable/ .. _Cookiecutter: https://cookiecutter.readthedocs.io/en/stable/ -.. _Mypy: https://mypy.readthedocs.io/en/stable/ .. _Coverage: https://coverage.readthedocs.io/en/7.13.4/ -.. _Bandit: https://bandit.readthedocs.io/en/latest/ -.. _Sphinx: https://www.sphinx-doc.org/en/master +.. _Mypy: https://mypy.readthedocs.io/en/stable/ +.. _Nox: https://nox.thea.codes/en/stable/ +.. _Pip Audit: https://pypi.org/project/pip-audit/ +.. _Poetry: https://python-poetry.org +.. _Pylint: https://pylint.readthedocs.io/en/stable/ +.. _Ruff: https://docs.astral.sh/ruff .. _Sonar: https://docs.sonarsource.com/sonarqube-server +.. _Sphinx: https://www.sphinx-doc.org/en/master diff --git a/exasol/toolbox/nox/_dependencies.py b/exasol/toolbox/nox/_dependencies.py index d196dfd08..c6c64401e 100644 --- a/exasol/toolbox/nox/_dependencies.py +++ b/exasol/toolbox/nox/_dependencies.py @@ -32,8 +32,7 @@ def dependency_licenses(session: Session) -> None: print(license_markdown.to_markdown()) -# Probably this session is obsolete -@nox.session(name="dependency:audit-old", python=False) +@nox.session(name="dependency:audit", python=False) def audit(session: Session) -> None: """Report known vulnerabilities.""" From 0ace33ff5834d212510b38dcdf3ed208da5bcbb7 Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 10 Apr 2026 12:39:20 +0200 Subject: [PATCH 15/37] merged changes from markdown.py --- exasol/toolbox/util/release/markdown.py | 8 +- test/unit/util/release/test_markdown.py | 173 ++++++++++++++++-------- 2 files changed, 122 insertions(+), 59 deletions(-) diff --git a/exasol/toolbox/util/release/markdown.py b/exasol/toolbox/util/release/markdown.py index 68d40248b..6bd9eb0c4 100644 --- a/exasol/toolbox/util/release/markdown.py +++ b/exasol/toolbox/util/release/markdown.py @@ -129,7 +129,13 @@ def elements(): return "\n\n".join(e for e in elements() if e) def __eq__(self, other) -> bool: - return isinstance(other, Markdown) and self.rendered == other.rendered + return ( + isinstance(other, Markdown) + and other.title == self.title + and other.intro == self.intro + and other.items == self.items + and other.children == self.children + ) def __str__(self) -> str: return self.rendered diff --git a/test/unit/util/release/test_markdown.py b/test/unit/util/release/test_markdown.py index a6916439e..b3fba3956 100644 --- a/test/unit/util/release/test_markdown.py +++ b/test/unit/util/release/test_markdown.py @@ -9,6 +9,10 @@ ) +def _markdown(text: str) -> Markdown: + return Markdown.from_text(cleandoc(text)) + + class Scenario: def __init__( self, initial: str, expected_output: str, expected_children: list[str] @@ -21,14 +25,6 @@ def create_testee(self) -> Markdown: return Markdown.from_text(self.initial) -INVALID_MARKDOWN = cleandoc(""" - # Title - - Some text. - - # Another Title - """) - MINIMAL = Scenario( initial=""" # title @@ -42,27 +38,31 @@ def create_testee(self) -> Markdown: expected_children=[], ) -WITH_CHILD = Scenario( +FULL = Scenario( initial=""" - # Parent - text + # title + intro + * item one + * item two ## Child - paragraph - - * item 1 - * item 2 + cintro + - item c1 + - item c2 """, expected_output=""" - # Parent + # title - text + intro + + * item one + * item two ## Child - paragraph + cintro - * item 1 - * item 2 + - item c1 + - item c2 """, expected_children=["## Child"], ) @@ -124,15 +124,15 @@ def create_testee(self) -> Markdown: expected_children=["## Child A", "## Child B"], ) +CHILD = _markdown(""" + ## Sample Child + child intro. + """) -@pytest.fixture -def sample_child() -> Markdown: - return Markdown(title="## New", intro="intro") - - -@pytest.fixture -def illegal_child() -> Markdown: - return Markdown(title="# Top-level", intro="intro") +ILLEGAL_CHILD = _markdown(""" + # Top-level + intro + """) def test_no_title_error(): @@ -141,33 +141,90 @@ def test_no_title_error(): def test_additional_line_error(): + invalid_markdown = cleandoc(""" + # Title + Some text. + # Another Title + """) + expected_error = ( 'additional line "# Another Title" after top-level section "# Title".' ) with pytest.raises(ParseError, match=expected_error): - Markdown.from_text(INVALID_MARKDOWN) + Markdown.from_text(invalid_markdown) -def test_constructor_illegal_child(illegal_child: Markdown): +def test_constructor_illegal_child(): with pytest.raises(IllegalChild): - Markdown("# title", children=[illegal_child]) + Markdown("# title", children=[ILLEGAL_CHILD]) -def test_no_intro() -> None: - content = """ - # title - * #123: Fixed vulnerability - """ - testee = Markdown.from_text(cleandoc(content)) - assert testee == Markdown("# title", "", "* #123: Fixed vulnerability") +@pytest.mark.parametrize( + "content, expected", + [ + pytest.param( + """ + # title + """, + Markdown("# title"), + id="only_title", + ), + pytest.param( + """ + # title + intro + """, + Markdown("# title", "intro"), + id="intro", + ), + pytest.param( + """ + # title + * item 1 + """, + Markdown("# title", "", "* item 1"), + id="items", + ), + pytest.param( + """ + # title + intro + * item 1 + * item 2 + """, + Markdown("# title", "intro", "* item 1\n* item 2"), + id="intro_and_items", + ), + pytest.param( + """ + # title + intro + - item 1 + - item 2 + """, + Markdown("# title", "intro", "- item 1\n- item 2"), + id="intro_dash_items", + ), + ], +) +def test_equals(content: str, expected: Markdown) -> None: + assert Markdown.from_text(cleandoc(content)) == expected -def test_equals() -> None: - testee = MINIMAL.create_testee() - other = MINIMAL.create_testee() - assert other == testee - other.title = "# other" - assert other != testee +@pytest.mark.parametrize( + "attr, value", + [ + ("title", "# other"), + ("intro", "other"), + ("items", "- aaa"), + ("children", []), + ], +) +def test_different(attr, value) -> None: + testee = FULL.create_testee() + other = FULL.create_testee() + setattr(other, attr, value) + assert testee != other def test_test_read(tmp_path) -> None: @@ -176,7 +233,7 @@ def test_test_read(tmp_path) -> None: assert Markdown.read(file) == MINIMAL.create_testee() -ALL_SCENARIOS = [MINIMAL, WITH_CHILD, TWO_CHILDREN, NESTED] +ALL_SCENARIOS = [MINIMAL, FULL, TWO_CHILDREN, NESTED] @pytest.mark.parametrize("scenario", ALL_SCENARIOS) @@ -203,25 +260,25 @@ def test_rendered(scenario: Scenario): "scenario, pos", [ (MINIMAL, 0), - (WITH_CHILD, 1), + (FULL, 1), (TWO_CHILDREN, 1), ], ) -def test_add_child(sample_child: Markdown, scenario: Scenario, pos: int): +def test_add_child(scenario: Scenario, pos: int): testee = scenario.create_testee() - testee.add_child(sample_child) - assert testee.children[pos] == sample_child + testee.add_child(CHILD) + assert testee.children[pos] == CHILD -def test_replace_illegal_child(illegal_child): - testee = WITH_CHILD.create_testee() +def test_replace_illegal_child(): + testee = FULL.create_testee() with pytest.raises(IllegalChild): - testee.replace_or_append_child(illegal_child) + testee.replace_or_append_child(ILLEGAL_CHILD) @pytest.mark.parametrize("scenario", ALL_SCENARIOS) def test_replace_existing_child(scenario: Scenario): - testee = WITH_CHILD.create_testee() + testee = FULL.create_testee() old_child = testee.children[0] old_rendered = testee.rendered new_child = Markdown(old_child.title, "new intro") @@ -231,19 +288,19 @@ def test_replace_existing_child(scenario: Scenario): @pytest.mark.parametrize("scenario", ALL_SCENARIOS) -def test_replace_non_existing_child(scenario: Scenario, sample_child: Markdown): +def test_replace_non_existing_child(scenario: Scenario): testee = scenario.create_testee() expected = len(testee.children) + 1 - testee.replace_or_append_child(sample_child) + testee.replace_or_append_child(CHILD) assert len(testee.children) == expected - assert testee.children[-1] == sample_child + assert testee.children[-1] == CHILD @pytest.mark.parametrize("scenario", ALL_SCENARIOS) -def test_add_illegal_child(illegal_child: Markdown, scenario: Scenario): +def test_add_illegal_child(scenario: Scenario): testee = scenario.create_testee() with pytest.raises(IllegalChild): - testee.add_child(illegal_child) + testee.add_child(ILLEGAL_CHILD) def test_nested(): From b8b0a0f6916e5b1c921a152da5fa484f6c1b8c3c Mon Sep 17 00:00:00 2001 From: ckunki Date: Sun, 12 Apr 2026 13:15:05 +0200 Subject: [PATCH 16/37] merged changes from markdown.py --- doc/changes/changes_0.15.0.md | 4 ++-- doc/changes/unreleased.md | 18 +++--------------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/doc/changes/changes_0.15.0.md b/doc/changes/changes_0.15.0.md index 7188f56b9..f937252ec 100644 --- a/doc/changes/changes_0.15.0.md +++ b/doc/changes/changes_0.15.0.md @@ -24,5 +24,5 @@ ## 🔩 Internal -* Update depdency constraints -* Relock dependencies \ No newline at end of file +* Update dependency constraints +* Relock dependencies diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 920f82f7a..16d43ad13 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -2,21 +2,9 @@ ## Summary -This release fixes vulnerabilities by updating transitive dependencies in the `poetry.lock` file. - -| Dependency | Version | ID | Fix Versions | Updated to | -|--------------|---------|----------------|--------------|------------| -| cryptography | 46.0.5 | CVE-2026-34073 | 46.0.6 | 46.0.6 | -| pygments | 2.19.2 | CVE-2026-4539 | 2.20.0 | 2.20.0 | -| requests | 2.32.5 | CVE-2026-25645 | 2.33.0 | 2.33.1 | - -To ensure usage of secure packages, it is up to the user to similarly relock their dependencies. +This release adds nox session `vulnerabilities:resolved` reporting resolved +GitHub security issues since the last release. ## Features -* #740: Added nox session `release:update` -* #402: Created nox task to detect resolved GitHub security issues - -## Security Issues - -* #759: Fixed vulnerabilities by re-locking transitive dependencies & updated `actions/deploy-pages` from v4 to v5 +* #402: Created nox session `vulnerabilities:resolved` to report resolved GitHub security issues From 4fa56b822042d2ad7c43607ec1aa297649f0170c Mon Sep 17 00:00:00 2001 From: ckunki Date: Sun, 12 Apr 2026 13:16:41 +0200 Subject: [PATCH 17/37] merged changes from markdown.py --- .github/actions/security-issues/action.yml | 2 +- doc/changes/changelog.md | 2 ++ doc/changes/changes_0.15.0.md | 4 ++-- doc/changes/changes_6.2.0.md | 28 ++++++++++++++++++++++ 4 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 doc/changes/changes_6.2.0.md diff --git a/.github/actions/security-issues/action.yml b/.github/actions/security-issues/action.yml index 1a857aa9c..7f7fe5f06 100644 --- a/.github/actions/security-issues/action.yml +++ b/.github/actions/security-issues/action.yml @@ -39,7 +39,7 @@ runs: - name: Install Python Toolbox / Security tool shell: bash run: | - pip install exasol-toolbox==6.1.1 + pip install exasol-toolbox==6.2.0 - name: Create Security Issue Report shell: bash diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index 7b0d6eb9c..cef209e78 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,6 +1,7 @@ # Changelog * [unreleased](unreleased.md) +* [6.2.0](changes_6.2.0.md) * [6.1.1](changes_6.1.1.md) * [6.1.0](changes_6.1.0.md) * [6.0.0](changes_6.0.0.md) @@ -60,6 +61,7 @@ hidden: --- unreleased +changes_6.2.0 changes_6.1.1 changes_6.1.0 changes_6.0.0 diff --git a/doc/changes/changes_0.15.0.md b/doc/changes/changes_0.15.0.md index f937252ec..7188f56b9 100644 --- a/doc/changes/changes_0.15.0.md +++ b/doc/changes/changes_0.15.0.md @@ -24,5 +24,5 @@ ## 🔩 Internal -* Update dependency constraints -* Relock dependencies +* Update depdency constraints +* Relock dependencies \ No newline at end of file diff --git a/doc/changes/changes_6.2.0.md b/doc/changes/changes_6.2.0.md new file mode 100644 index 000000000..08dfb81a9 --- /dev/null +++ b/doc/changes/changes_6.2.0.md @@ -0,0 +1,28 @@ +# 6.2.0 - 2026-04-10 + +## Summary + +This release fixes vulnerabilities by updating transitive dependencies in the `poetry.lock` file. + +| Dependency | Version | ID | Fix Versions | Updated to | +|--------------|---------|----------------|--------------|------------| +| cryptography | 46.0.5 | CVE-2026-34073 | 46.0.6 | 46.0.7 | +| cryptography | 46.0.6 | CVE-2026-39892 | 46.0.7 | 46.0.7 | +| pygments | 2.19.2 | CVE-2026-4539 | 2.20.0 | 2.20.0 | +| requests | 2.32.5 | CVE-2026-25645 | 2.33.0 | 2.33.1 | + +To ensure usage of secure packages, it is up to the user to similarly relock their dependencies. + +## Features + +* #740: Added nox session `release:update` + +## Security Issues + +* #759: Fixed vulnerabilities by re-locking transitive dependencies & updated `actions/deploy-pages` from v4 to v5 + +## Dependency Updates + +### `main` + +* Updated dependency `pysonar:1.3.0.4086` to `1.0.2.1722` From daea685b0c603cbb58194009317de150a4c66dcc Mon Sep 17 00:00:00 2001 From: ckunki Date: Sun, 12 Apr 2026 13:18:04 +0200 Subject: [PATCH 18/37] merged changes from markdown.py --- doc/changes/changes_0.15.0.md | 4 +- doc/user_guide/dependencies.rst | 2 +- .../features/managing_dependencies.rst | 2 +- exasol/toolbox/version.py | 4 +- poetry.lock | 104 +++++++++--------- project-template/cookiecutter.json | 4 +- pyproject.toml | 2 +- 7 files changed, 61 insertions(+), 61 deletions(-) diff --git a/doc/changes/changes_0.15.0.md b/doc/changes/changes_0.15.0.md index 7188f56b9..f937252ec 100644 --- a/doc/changes/changes_0.15.0.md +++ b/doc/changes/changes_0.15.0.md @@ -24,5 +24,5 @@ ## 🔩 Internal -* Update depdency constraints -* Relock dependencies \ No newline at end of file +* Update dependency constraints +* Relock dependencies diff --git a/doc/user_guide/dependencies.rst b/doc/user_guide/dependencies.rst index ab1be3797..d37f14a3e 100644 --- a/doc/user_guide/dependencies.rst +++ b/doc/user_guide/dependencies.rst @@ -56,7 +56,7 @@ system-wide Poetry installation to most effectively use Poetry ``2.3.0``: * `PEP-735 `__ .. note:: - Note that `uvx migrate-to-uv `__ + Note that `uvx migrate-to-uv `__ seems to do a good job with automating many of the PEP-related changes. Though, a developer should take care to verify the changes, as some are unneeded as it completes the migration to ``uv`` which the PTB does NOT yet support. diff --git a/doc/user_guide/features/managing_dependencies.rst b/doc/user_guide/features/managing_dependencies.rst index a89b52735..cc4a349a7 100644 --- a/doc/user_guide/features/managing_dependencies.rst +++ b/doc/user_guide/features/managing_dependencies.rst @@ -11,7 +11,7 @@ Managing Dependencies and Vulnerabilities | | | vulnerabilities in our dependencies | +------------------------------+----------------+-------------------------------------+ | ``vulnerabilities:resolved`` | No | Uses ``pip-audit`` to report known | -| | | vulnerabilities in depdendencies | +| | | vulnerabilities in dependencies | | | | that have been resolved in | | | | comparison to the last release. | +------------------------------+----------------+-------------------------------------+ diff --git a/exasol/toolbox/version.py b/exasol/toolbox/version.py index 46c89c6a8..dfa023387 100644 --- a/exasol/toolbox/version.py +++ b/exasol/toolbox/version.py @@ -9,7 +9,7 @@ """ MAJOR = 6 -MINOR = 1 -PATCH = 1 +MINOR = 2 +PATCH = 0 VERSION = f"{MAJOR}.{MINOR}.{PATCH}" __version__ = VERSION diff --git a/poetry.lock b/poetry.lock index 0ef82c8a6..c21dc8dbd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.3 and should not be changed by hand. [[package]] name = "accessible-pygments" @@ -842,62 +842,62 @@ toml = ["tomli ; python_full_version <= \"3.11.0a6\""] [[package]] name = "cryptography" -version = "46.0.6" +version = "46.0.7" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = "!=3.9.0,!=3.9.1,>=3.8" groups = ["main"] markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and sys_platform == \"linux\"" files = [ - {file = "cryptography-46.0.6-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:64235194bad039a10bb6d2d930ab3323baaec67e2ce36215fd0952fad0930ca8"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:26031f1e5ca62fcb9d1fcb34b2b60b390d1aacaa15dc8b895a9ed00968b97b30"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9a693028b9cbe51b5a1136232ee8f2bc242e4e19d456ded3fa7c86e43c713b4a"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:67177e8a9f421aa2d3a170c3e56eca4e0128883cf52a071a7cbf53297f18b175"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:d9528b535a6c4f8ff37847144b8986a9a143585f0540fbcb1a98115b543aa463"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:22259338084d6ae497a19bae5d4c66b7ca1387d3264d1c2c0e72d9e9b6a77b97"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:760997a4b950ff00d418398ad73fbc91aa2894b5c1db7ccb45b4f68b42a63b3c"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:3dfa6567f2e9e4c5dceb8ccb5a708158a2a871052fa75c8b78cb0977063f1507"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:cdcd3edcbc5d55757e5f5f3d330dd00007ae463a7e7aa5bf132d1f22a4b62b19"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:d4e4aadb7fc1f88687f47ca20bb7227981b03afaae69287029da08096853b738"}, - {file = "cryptography-46.0.6-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2b417edbe8877cda9022dde3a008e2deb50be9c407eef034aeeb3a8b11d9db3c"}, - {file = "cryptography-46.0.6-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:380343e0653b1c9d7e1f55b52aaa2dbb2fdf2730088d48c43ca1c7c0abb7cc2f"}, - {file = "cryptography-46.0.6-cp311-abi3-win32.whl", hash = "sha256:bcb87663e1f7b075e48c3be3ecb5f0b46c8fc50b50a97cf264e7f60242dca3f2"}, - {file = "cryptography-46.0.6-cp311-abi3-win_amd64.whl", hash = "sha256:6739d56300662c468fddb0e5e291f9b4d084bead381667b9e654c7dd81705124"}, - {file = "cryptography-46.0.6-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:2ef9e69886cbb137c2aef9772c2e7138dc581fad4fcbcf13cc181eb5a3ab6275"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7f417f034f91dcec1cb6c5c35b07cdbb2ef262557f701b4ecd803ee8cefed4f4"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d24c13369e856b94892a89ddf70b332e0b70ad4a5c43cf3e9cb71d6d7ffa1f7b"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:aad75154a7ac9039936d50cf431719a2f8d4ed3d3c277ac03f3339ded1a5e707"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:3c21d92ed15e9cfc6eb64c1f5a0326db22ca9c2566ca46d845119b45b4400361"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:4668298aef7cddeaf5c6ecc244c2302a2b8e40f384255505c22875eebb47888b"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:8ce35b77aaf02f3b59c90b2c8a05c73bac12cea5b4e8f3fbece1f5fddea5f0ca"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:c89eb37fae9216985d8734c1afd172ba4927f5a05cfd9bf0e4863c6d5465b013"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:ed418c37d095aeddf5336898a132fba01091f0ac5844e3e8018506f014b6d2c4"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:69cf0056d6947edc6e6760e5f17afe4bea06b56a9ac8a06de9d2bd6b532d4f3a"}, - {file = "cryptography-46.0.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8e7304c4f4e9490e11efe56af6713983460ee0780f16c63f219984dab3af9d2d"}, - {file = "cryptography-46.0.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b928a3ca837c77a10e81a814a693f2295200adb3352395fad024559b7be7a736"}, - {file = "cryptography-46.0.6-cp314-cp314t-win32.whl", hash = "sha256:97c8115b27e19e592a05c45d0dd89c57f81f841cc9880e353e0d3bf25b2139ed"}, - {file = "cryptography-46.0.6-cp314-cp314t-win_amd64.whl", hash = "sha256:c797e2517cb7880f8297e2c0f43bb910e91381339336f75d2c1c2cbf811b70b4"}, - {file = "cryptography-46.0.6-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:12cae594e9473bca1a7aceb90536060643128bb274fcea0fc459ab90f7d1ae7a"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:639301950939d844a9e1c4464d7e07f902fe9a7f6b215bb0d4f28584729935d8"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ed3775295fb91f70b4027aeba878d79b3e55c0b3e97eaa4de71f8f23a9f2eb77"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:8927ccfbe967c7df312ade694f987e7e9e22b2425976ddbf28271d7e58845290"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:b12c6b1e1651e42ab5de8b1e00dc3b6354fdfd778e7fa60541ddacc27cd21410"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:063b67749f338ca9c5a0b7fe438a52c25f9526b851e24e6c9310e7195aad3b4d"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:02fad249cb0e090b574e30b276a3da6a149e04ee2f049725b1f69e7b8351ec70"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:7e6142674f2a9291463e5e150090b95a8519b2fb6e6aaec8917dd8d094ce750d"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:456b3215172aeefb9284550b162801d62f5f264a081049a3e94307fe20792cfa"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:341359d6c9e68834e204ceaf25936dffeafea3829ab80e9503860dcc4f4dac58"}, - {file = "cryptography-46.0.6-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9a9c42a2723999a710445bc0d974e345c32adfd8d2fac6d8a251fa829ad31cfb"}, - {file = "cryptography-46.0.6-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6617f67b1606dfd9fe4dbfa354a9508d4a6d37afe30306fe6c101b7ce3274b72"}, - {file = "cryptography-46.0.6-cp38-abi3-win32.whl", hash = "sha256:7f6690b6c55e9c5332c0b59b9c8a3fb232ebf059094c17f9019a51e9827df91c"}, - {file = "cryptography-46.0.6-cp38-abi3-win_amd64.whl", hash = "sha256:79e865c642cfc5c0b3eb12af83c35c5aeff4fa5c672dc28c43721c2c9fdd2f0f"}, - {file = "cryptography-46.0.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:2ea0f37e9a9cf0df2952893ad145fd9627d326a59daec9b0802480fa3bcd2ead"}, - {file = "cryptography-46.0.6-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a3e84d5ec9ba01f8fd03802b2147ba77f0c8f2617b2aff254cedd551844209c8"}, - {file = "cryptography-46.0.6-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:12f0fa16cc247b13c43d56d7b35287ff1569b5b1f4c5e87e92cc4fcc00cd10c0"}, - {file = "cryptography-46.0.6-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:50575a76e2951fe7dbd1f56d181f8c5ceeeb075e9ff88e7ad997d2f42af06e7b"}, - {file = "cryptography-46.0.6-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:90e5f0a7b3be5f40c3a0a0eafb32c681d8d2c181fc2a1bdabe9b3f611d9f6b1a"}, - {file = "cryptography-46.0.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:6728c49e3b2c180ef26f8e9f0a883a2c585638db64cf265b49c9ba10652d430e"}, - {file = "cryptography-46.0.6.tar.gz", hash = "sha256:27550628a518c5c6c903d84f637fbecf287f6cb9ced3804838a1295dc1fd0759"}, + {file = "cryptography-46.0.7-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:ea42cbe97209df307fdc3b155f1b6fa2577c0defa8f1f7d3be7d31d189108ad4"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b36a4695e29fe69215d75960b22577197aca3f7a25b9cf9d165dcfe9d80bc325"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5ad9ef796328c5e3c4ceed237a183f5d41d21150f972455a9d926593a1dcb308"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:73510b83623e080a2c35c62c15298096e2a5dc8d51c3b4e1740211839d0dea77"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:cbd5fb06b62bd0721e1170273d3f4d5a277044c47ca27ee257025146c34cbdd1"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:420b1e4109cc95f0e5700eed79908cef9268265c773d3a66f7af1eef53d409ef"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:24402210aa54baae71d99441d15bb5a1919c195398a87b563df84468160a65de"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:8a469028a86f12eb7d2fe97162d0634026d92a21f3ae0ac87ed1c4a447886c83"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9694078c5d44c157ef3162e3bf3946510b857df5a3955458381d1c7cfc143ddb"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:42a1e5f98abb6391717978baf9f90dc28a743b7d9be7f0751a6f56a75d14065b"}, + {file = "cryptography-46.0.7-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:91bbcb08347344f810cbe49065914fe048949648f6bd5c2519f34619142bbe85"}, + {file = "cryptography-46.0.7-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5d1c02a14ceb9148cc7816249f64f623fbfee39e8c03b3650d842ad3f34d637e"}, + {file = "cryptography-46.0.7-cp311-abi3-win32.whl", hash = "sha256:d23c8ca48e44ee015cd0a54aeccdf9f09004eba9fc96f38c911011d9ff1bd457"}, + {file = "cryptography-46.0.7-cp311-abi3-win_amd64.whl", hash = "sha256:397655da831414d165029da9bc483bed2fe0e75dde6a1523ec2fe63f3c46046b"}, + {file = "cryptography-46.0.7-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:d151173275e1728cf7839aaa80c34fe550c04ddb27b34f48c232193df8db5842"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:db0f493b9181c7820c8134437eb8b0b4792085d37dbb24da050476ccb664e59c"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ebd6daf519b9f189f85c479427bbd6e9c9037862cf8fe89ee35503bd209ed902"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b7b412817be92117ec5ed95f880defe9cf18a832e8cafacf0a22337dc1981b4d"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:fbfd0e5f273877695cb93baf14b185f4878128b250cc9f8e617ea0c025dfb022"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:ffca7aa1d00cf7d6469b988c581598f2259e46215e0140af408966a24cf086ce"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:60627cf07e0d9274338521205899337c5d18249db56865f943cbe753aa96f40f"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:80406c3065e2c55d7f49a9550fe0c49b3f12e5bfff5dedb727e319e1afb9bf99"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:c5b1ccd1239f48b7151a65bc6dd54bcfcc15e028c8ac126d3fada09db0e07ef1"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:d5f7520159cd9c2154eb61eb67548ca05c5774d39e9c2c4339fd793fe7d097b2"}, + {file = "cryptography-46.0.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fcd8eac50d9138c1d7fc53a653ba60a2bee81a505f9f8850b6b2888555a45d0e"}, + {file = "cryptography-46.0.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:65814c60f8cc400c63131584e3e1fad01235edba2614b61fbfbfa954082db0ee"}, + {file = "cryptography-46.0.7-cp314-cp314t-win32.whl", hash = "sha256:fdd1736fed309b4300346f88f74cd120c27c56852c3838cab416e7a166f67298"}, + {file = "cryptography-46.0.7-cp314-cp314t-win_amd64.whl", hash = "sha256:e06acf3c99be55aa3b516397fe42f5855597f430add9c17fa46bf2e0fb34c9bb"}, + {file = "cryptography-46.0.7-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:462ad5cb1c148a22b2e3bcc5ad52504dff325d17daf5df8d88c17dda1f75f2a4"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:84d4cced91f0f159a7ddacad249cc077e63195c36aac40b4150e7a57e84fffe7"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:128c5edfe5e5938b86b03941e94fac9ee793a94452ad1365c9fc3f4f62216832"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5e51be372b26ef4ba3de3c167cd3d1022934bc838ae9eaad7e644986d2a3d163"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:cdf1a610ef82abb396451862739e3fc93b071c844399e15b90726ef7470eeaf2"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1d25aee46d0c6f1a501adcddb2d2fee4b979381346a78558ed13e50aa8a59067"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:cdfbe22376065ffcf8be74dc9a909f032df19bc58a699456a21712d6e5eabfd0"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:abad9dac36cbf55de6eb49badd4016806b3165d396f64925bf2999bcb67837ba"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:935ce7e3cfdb53e3536119a542b839bb94ec1ad081013e9ab9b7cfd478b05006"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:35719dc79d4730d30f1c2b6474bd6acda36ae2dfae1e3c16f2051f215df33ce0"}, + {file = "cryptography-46.0.7-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7bbc6ccf49d05ac8f7d7b5e2e2c33830d4fe2061def88210a126d130d7f71a85"}, + {file = "cryptography-46.0.7-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a1529d614f44b863a7b480c6d000fe93b59acee9c82ffa027cfadc77521a9f5e"}, + {file = "cryptography-46.0.7-cp38-abi3-win32.whl", hash = "sha256:f247c8c1a1fb45e12586afbb436ef21ff1e80670b2861a90353d9b025583d246"}, + {file = "cryptography-46.0.7-cp38-abi3-win_amd64.whl", hash = "sha256:506c4ff91eff4f82bdac7633318a526b1d1309fc07ca76a3ad182cb5b686d6d3"}, + {file = "cryptography-46.0.7-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:fc9ab8856ae6cf7c9358430e49b368f3108f050031442eaeb6b9d87e4dcf4e4f"}, + {file = "cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d3b99c535a9de0adced13d159c5a9cf65c325601aa30f4be08afd680643e9c15"}, + {file = "cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d02c738dacda7dc2a74d1b2b3177042009d5cab7c7079db74afc19e56ca1b455"}, + {file = "cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:04959522f938493042d595a736e7dbdff6eb6cc2339c11465b3ff89343b65f65"}, + {file = "cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:3986ac1dee6def53797289999eabe84798ad7817f3e97779b5061a95b0ee4968"}, + {file = "cryptography-46.0.7-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:258514877e15963bd43b558917bc9f54cf7cf866c38aa576ebf47a77ddbc43a4"}, + {file = "cryptography-46.0.7.tar.gz", hash = "sha256:e4cfd68c5f3e0bfdad0d38e023239b96a2fe84146481852dffbcca442c245aa5"}, ] [package.dependencies] @@ -911,7 +911,7 @@ nox = ["nox[uv] (>=2024.4.15)"] pep8test = ["check-sdist", "click (>=8.0.1)", "mypy (>=1.14)", "ruff (>=0.11.11)"] sdist = ["build (>=1.0.0)"] ssh = ["bcrypt (>=3.1.5)"] -test = ["certifi (>=2024)", "cryptography-vectors (==46.0.6)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] +test = ["certifi (>=2024)", "cryptography-vectors (==46.0.7)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] test-randomorder = ["pytest-randomly"] [[package]] diff --git a/project-template/cookiecutter.json b/project-template/cookiecutter.json index c30b80a3f..5b3db10b6 100644 --- a/project-template/cookiecutter.json +++ b/project-template/cookiecutter.json @@ -9,11 +9,11 @@ "author_email": "opensource@exasol.com", "project_short_tag": "", "python_version_min": "3.10", - "exasol_toolbox_version_range": ">=6.1.1,<7", + "exasol_toolbox_version_range": ">=6.2.0,<7", "license_year": "{% now 'utc', '%Y' %}", "__repo_name_slug": "{{cookiecutter.package_name}}", "__package_name_slug": "{{cookiecutter.package_name}}", "_extensions": [ "cookiecutter.extensions.TimeExtension" ] -} \ No newline at end of file +} diff --git a/pyproject.toml b/pyproject.toml index 3fe0f1cae..5424c1910 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "exasol-toolbox" -version = "6.1.1" +version = "6.2.0" description = "Your one-stop solution for managing all standard tasks and core workflows of your Python project." authors = [ { name = "Nicola Coretti", email = "nicola.coretti@exasol.com" }, From 47af6f36df37c028438e2237b33cc82a52753356 Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 17 Apr 2026 12:28:43 +0200 Subject: [PATCH 19/37] simplified set comparisson --- exasol/toolbox/util/dependencies/track_vulnerabilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol/toolbox/util/dependencies/track_vulnerabilities.py b/exasol/toolbox/util/dependencies/track_vulnerabilities.py index dc084e384..bba07a1bb 100644 --- a/exasol/toolbox/util/dependencies/track_vulnerabilities.py +++ b/exasol/toolbox/util/dependencies/track_vulnerabilities.py @@ -37,7 +37,7 @@ def is_resolved(self, vuln: Vulnerability) -> bool: """ refs = set(vuln.references) current = self._references.get(vuln.package.name, set()) - return not refs.intersection(current) + return refs.isdisjoint(current) class DependenciesAudit(BaseModel): From 0960edb523275255f1830d53d8a9e4c8941da0f2 Mon Sep 17 00:00:00 2001 From: ckunki Date: Sat, 18 Apr 2026 11:05:51 +0200 Subject: [PATCH 20/37] updated changelog --- doc/changes/unreleased.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 441aa7906..fe3f77f36 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -5,6 +5,7 @@ ## Features * #777: Improved VulnerabilityMatcher to handle packages with multiple vulnerabilities +* #517: Modified nox session `release:prepare` to report resolved security issues ## Refactoring From 52a68318ecd6aa188039f1ebdbcff2ed636a877d Mon Sep 17 00:00:00 2001 From: ckunki Date: Mon, 20 Apr 2026 14:32:50 +0200 Subject: [PATCH 21/37] Added missing import --- exasol/toolbox/util/release/changelog.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/exasol/toolbox/util/release/changelog.py b/exasol/toolbox/util/release/changelog.py index 8db96f8fa..d5ab0bdab 100644 --- a/exasol/toolbox/util/release/changelog.py +++ b/exasol/toolbox/util/release/changelog.py @@ -1,10 +1,13 @@ from __future__ import annotations +import datetime import re from collections import OrderedDict +from collections.abc import Generator from datetime import datetime from inspect import cleandoc from pathlib import Path +from typing import Dict from exasol.toolbox.util.dependencies.audit import ( get_vulnerabilities, From 9da0a7f6255d3b2ddf6ddbcdd3b15060ee356941 Mon Sep 17 00:00:00 2001 From: ckunki Date: Mon, 20 Apr 2026 14:34:34 +0200 Subject: [PATCH 22/37] nox -s format:fix --- .../util/dependencies/track_vulnerabilities.py | 15 +++++++++------ exasol/toolbox/util/release/changelog.py | 11 ++++------- test/unit/util/release/changelog_test.py | 10 +++++++--- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/exasol/toolbox/util/dependencies/track_vulnerabilities.py b/exasol/toolbox/util/dependencies/track_vulnerabilities.py index bba07a1bb..d9b104dee 100644 --- a/exasol/toolbox/util/dependencies/track_vulnerabilities.py +++ b/exasol/toolbox/util/dependencies/track_vulnerabilities.py @@ -1,4 +1,3 @@ -from collections import defaultdict from inspect import cleandoc from pydantic import ( @@ -15,8 +14,7 @@ def __init__(self, current_vulnerabilities: list[Vulnerability]): # * keys: package names # * values: set of each vulnerability's references self._references = { - v.package.name: set(v.references) - for v in current_vulnerabilities + v.package.name: set(v.references) for v in current_vulnerabilities } def is_resolved(self, vuln: Vulnerability) -> bool: @@ -58,8 +56,7 @@ def resolved_vulnerabilities(self) -> list[Vulnerability]: """ matcher = VulnerabilityMatcher(self.current_vulnerabilities) return [ - vuln for vuln in self.previous_vulnerabilities - if matcher.is_resolved(vuln) + vuln for vuln in self.previous_vulnerabilities if matcher.is_resolved(vuln) ] def report_resolved_vulnerabilities(self) -> str: @@ -74,8 +71,14 @@ def report_resolved_vulnerabilities(self) -> str: |------------|---------------|----------|----------| """ ) + def formatted(vuln: Vulnerability) -> str: - columns = (vuln.package.name, vuln.id, str(vuln.package.version), vuln.fix_versions[0]) + columns = ( + vuln.package.name, + vuln.id, + str(vuln.package.version), + vuln.fix_versions[0], + ) return f'| {" | ".join(columns)} |' body = "\n".join(formatted(v) for v in resolved) diff --git a/exasol/toolbox/util/release/changelog.py b/exasol/toolbox/util/release/changelog.py index d5ab0bdab..381f03995 100644 --- a/exasol/toolbox/util/release/changelog.py +++ b/exasol/toolbox/util/release/changelog.py @@ -1,13 +1,10 @@ from __future__ import annotations -import datetime -import re from collections import OrderedDict from collections.abc import Generator from datetime import datetime from inspect import cleandoc from pathlib import Path -from typing import Dict from exasol.toolbox.util.dependencies.audit import ( get_vulnerabilities, @@ -62,9 +59,7 @@ def _dependency_sections(self) -> Generator[Markdown]: """ try: - previous_groups = get_dependencies_from_latest_tag( - root_path=self.root_path - ) + previous_groups = get_dependencies_from_latest_tag(root_path=self.root_path) except LatestTagNotFoundError: # In new projects, there is not a pre-existing tag, and all dependencies # are considered new. @@ -128,7 +123,9 @@ def get_changed_files(self) -> list[Path]: def _resolved_vulnerabilities(self) -> Markdown | None: report = DependenciesAudit( - previous_vulnerabilities=get_vulnerabilities_from_latest_tag(self.root_path), + previous_vulnerabilities=get_vulnerabilities_from_latest_tag( + self.root_path + ), current_vulnerabilities=get_vulnerabilities(self.root_path), ).report_resolved_vulnerabilities() return Markdown("## Security Issues", report) if report else None diff --git a/test/unit/util/release/changelog_test.py b/test/unit/util/release/changelog_test.py index 29b547044..42d5f3bbe 100644 --- a/test/unit/util/release/changelog_test.py +++ b/test/unit/util/release/changelog_test.py @@ -105,7 +105,7 @@ def expected_changes( if Expect.VULNERABILITIES in include: changes.add_child(vulnerabilities) if Expect.VULNERABILITIES_INSERTED in include: - changes.child("## Security Issues").intro = (vulnerabilities.intro) + changes.child("## Security Issues").intro = vulnerabilities.intro if Expect.DEPENDENCIES in include: changes.replace_or_append_child(dependencies) return changes @@ -301,7 +301,9 @@ def test_prepare_release( assert changelog.changelog.read_text() == SampleContent.new_changelog assert changelog.unreleased.read_text() == UNRELEASED_INITIAL_CONTENT versioned = Markdown.read(changelog.versioned_changes) - expected = expected_changes(include=Expect.DEPENDENCIES | Expect.VULNERABILITIES) + expected = expected_changes( + include=Expect.DEPENDENCIES | Expect.VULNERABILITIES + ) assert versioned == expected @staticmethod @@ -318,7 +320,9 @@ def test_update_latest( mock_changelog(dependency_changes, vulnerability_changes) changelog.update_latest() versioned = Markdown.read(changelog.versioned_changes) - expected = expected_changes(include=Expect.DEPENDENCIES | Expect.VULNERABILITIES) + expected = expected_changes( + include=Expect.DEPENDENCIES | Expect.VULNERABILITIES + ) assert versioned == expected @staticmethod From 27ace042f48b158e44f8871a749e1b4e25dda57a Mon Sep 17 00:00:00 2001 From: ckunki Date: Mon, 20 Apr 2026 14:38:39 +0200 Subject: [PATCH 23/37] Removed redundant file test_markdown.py --- test/unit/util/release/test_markdown.py | 308 ------------------------ 1 file changed, 308 deletions(-) delete mode 100644 test/unit/util/release/test_markdown.py diff --git a/test/unit/util/release/test_markdown.py b/test/unit/util/release/test_markdown.py deleted file mode 100644 index b3fba3956..000000000 --- a/test/unit/util/release/test_markdown.py +++ /dev/null @@ -1,308 +0,0 @@ -from inspect import cleandoc - -import pytest - -from exasol.toolbox.util.release.markdown import ( - IllegalChild, - Markdown, - ParseError, -) - - -def _markdown(text: str) -> Markdown: - return Markdown.from_text(cleandoc(text)) - - -class Scenario: - def __init__( - self, initial: str, expected_output: str, expected_children: list[str] - ): - self.initial = cleandoc(initial) - self.expected_output = cleandoc(expected_output) - self.expected_children = expected_children - - def create_testee(self) -> Markdown: - return Markdown.from_text(self.initial) - - -MINIMAL = Scenario( - initial=""" - # title - body - """, - expected_output=""" - # title - - body - """, - expected_children=[], -) - -FULL = Scenario( - initial=""" - # title - intro - * item one - * item two - ## Child - cintro - - item c1 - - item c2 - """, - expected_output=""" - # title - - intro - - * item one - * item two - - ## Child - - cintro - - - item c1 - - item c2 - """, - expected_children=["## Child"], -) - -TWO_CHILDREN = Scenario( - initial=""" - # Parent - text - ## C1 - aaa - ## C2 - bbb - """, - expected_output=""" - # Parent - - text - - ## C1 - - aaa - - ## C2 - - bbb - """, - expected_children=["## C1", "## C2"], -) - - -NESTED = Scenario( - initial=""" - # Parent - text - ## Child A - aaa - ### Grand Child - ccc - ## Child B - bbb - """, - expected_output=""" - # Parent - - text - - ## Child A - - aaa - - ### Grand Child - - ccc - - ## Child B - - bbb - """, - expected_children=["## Child A", "## Child B"], -) - -CHILD = _markdown(""" - ## Sample Child - child intro. - """) - -ILLEGAL_CHILD = _markdown(""" - # Top-level - intro - """) - - -def test_no_title_error(): - with pytest.raises(ParseError, match="First line of markdown file must be a title"): - Markdown.from_text("body\n# title") - - -def test_additional_line_error(): - invalid_markdown = cleandoc(""" - # Title - Some text. - # Another Title - """) - - expected_error = ( - 'additional line "# Another Title" after top-level section "# Title".' - ) - with pytest.raises(ParseError, match=expected_error): - Markdown.from_text(invalid_markdown) - - -def test_constructor_illegal_child(): - with pytest.raises(IllegalChild): - Markdown("# title", children=[ILLEGAL_CHILD]) - - -@pytest.mark.parametrize( - "content, expected", - [ - pytest.param( - """ - # title - """, - Markdown("# title"), - id="only_title", - ), - pytest.param( - """ - # title - intro - """, - Markdown("# title", "intro"), - id="intro", - ), - pytest.param( - """ - # title - * item 1 - """, - Markdown("# title", "", "* item 1"), - id="items", - ), - pytest.param( - """ - # title - intro - * item 1 - * item 2 - """, - Markdown("# title", "intro", "* item 1\n* item 2"), - id="intro_and_items", - ), - pytest.param( - """ - # title - intro - - item 1 - - item 2 - """, - Markdown("# title", "intro", "- item 1\n- item 2"), - id="intro_dash_items", - ), - ], -) -def test_equals(content: str, expected: Markdown) -> None: - assert Markdown.from_text(cleandoc(content)) == expected - - -@pytest.mark.parametrize( - "attr, value", - [ - ("title", "# other"), - ("intro", "other"), - ("items", "- aaa"), - ("children", []), - ], -) -def test_different(attr, value) -> None: - testee = FULL.create_testee() - other = FULL.create_testee() - setattr(other, attr, value) - assert testee != other - - -def test_test_read(tmp_path) -> None: - file = tmp_path / "sample.md" - file.write_text(MINIMAL.initial) - assert Markdown.read(file) == MINIMAL.create_testee() - - -ALL_SCENARIOS = [MINIMAL, FULL, TWO_CHILDREN, NESTED] - - -@pytest.mark.parametrize("scenario", ALL_SCENARIOS) -def test_number_of_children(scenario: Scenario): - assert len(scenario.create_testee().children) == len(scenario.expected_children) - - -@pytest.mark.parametrize("scenario", ALL_SCENARIOS) -def test_non_existing_child(scenario: Scenario): - assert scenario.create_testee().child("non existing") is None - - -@pytest.mark.parametrize("scenario", ALL_SCENARIOS) -def test_valid_child(scenario: Scenario): - assert all(scenario.create_testee().child(c) for c in scenario.expected_children) - - -@pytest.mark.parametrize("scenario", ALL_SCENARIOS) -def test_rendered(scenario: Scenario): - assert scenario.create_testee().rendered == scenario.expected_output - - -@pytest.mark.parametrize( - "scenario, pos", - [ - (MINIMAL, 0), - (FULL, 1), - (TWO_CHILDREN, 1), - ], -) -def test_add_child(scenario: Scenario, pos: int): - testee = scenario.create_testee() - testee.add_child(CHILD) - assert testee.children[pos] == CHILD - - -def test_replace_illegal_child(): - testee = FULL.create_testee() - with pytest.raises(IllegalChild): - testee.replace_or_append_child(ILLEGAL_CHILD) - - -@pytest.mark.parametrize("scenario", ALL_SCENARIOS) -def test_replace_existing_child(scenario: Scenario): - testee = FULL.create_testee() - old_child = testee.children[0] - old_rendered = testee.rendered - new_child = Markdown(old_child.title, "new intro") - expected = old_rendered.replace(old_child.rendered, new_child.rendered) - testee.replace_or_append_child(new_child) - assert testee.rendered == expected - - -@pytest.mark.parametrize("scenario", ALL_SCENARIOS) -def test_replace_non_existing_child(scenario: Scenario): - testee = scenario.create_testee() - expected = len(testee.children) + 1 - testee.replace_or_append_child(CHILD) - assert len(testee.children) == expected - assert testee.children[-1] == CHILD - - -@pytest.mark.parametrize("scenario", ALL_SCENARIOS) -def test_add_illegal_child(scenario: Scenario): - testee = scenario.create_testee() - with pytest.raises(IllegalChild): - testee.add_child(ILLEGAL_CHILD) - - -def test_nested(): - testee = NESTED.create_testee() - assert testee.child("## Child A").child("### Grand Child") is not None From 3c9f201a29193f72921840b434fb2e73d7397930 Mon Sep 17 00:00:00 2001 From: ckunki Date: Mon, 20 Apr 2026 15:07:26 +0200 Subject: [PATCH 24/37] Fixed tests --- .../dependencies/track_vulnerabilities.py | 13 ++- .../track_vulnerabilities_test.py | 79 ++++++++++++++++--- 2 files changed, 79 insertions(+), 13 deletions(-) diff --git a/exasol/toolbox/util/dependencies/track_vulnerabilities.py b/exasol/toolbox/util/dependencies/track_vulnerabilities.py index d9b104dee..e913a9496 100644 --- a/exasol/toolbox/util/dependencies/track_vulnerabilities.py +++ b/exasol/toolbox/util/dependencies/track_vulnerabilities.py @@ -13,9 +13,16 @@ def __init__(self, current_vulnerabilities: list[Vulnerability]): # Dict of current vulnerabilities: # * keys: package names # * values: set of each vulnerability's references - self._references = { - v.package.name: set(v.references) for v in current_vulnerabilities - } + # self._references = { + # v.package.name: set(v.references) for v in current_vulnerabilities + # } + self._references = {} + for v in current_vulnerabilities: + p = v.package.name + if not p in self._references: + self._references[p] = set(v.references) + entry = self._references[p] + self._references[p] = entry | set(v.references) def is_resolved(self, vuln: Vulnerability) -> bool: """ diff --git a/test/unit/util/dependencies/track_vulnerabilities_test.py b/test/unit/util/dependencies/track_vulnerabilities_test.py index cdfae905b..bff6eae71 100644 --- a/test/unit/util/dependencies/track_vulnerabilities_test.py +++ b/test/unit/util/dependencies/track_vulnerabilities_test.py @@ -1,3 +1,5 @@ +import pytest + from exasol.toolbox.util.dependencies.audit import Vulnerability from exasol.toolbox.util.dependencies.track_vulnerabilities import ( DependenciesAudit, @@ -5,11 +7,18 @@ ) -def _flip_id_and_alias(vulnerability: SampleVulnerability): - other = vulnerability +@pytest.fixture +def flipped_id_vulnerability(sample_vulnerability) -> Vulnerability: + """ + Returns an instance of SampleVulnerability equal to + sample_vulnerability() but with ID and first alias flipped to verify + handling of vulnerabilities with changed ID. + """ + + other = sample_vulnerability vuln_entry = { - "aliases": [other.vulnerability_id], - "id": other.cve_id, + "aliases": [other.cve_id], + "id": other.vulnerability_id, "fix_versions": other.vulnerability.fix_versions, "description": other.description, } @@ -20,22 +29,72 @@ def _flip_id_and_alias(vulnerability: SampleVulnerability): ) +PKG_DATA = {"name": "cryptography", "version": "46.0.6"} + +VULN_1 = Vulnerability( + package=PKG_DATA, + id="GHSA-m959-cc7f-wv43", + aliases=["CVE-2026-34073"], + fix_versions=["46.0.6"], + description="Dummy description", +) + +VULN_2 = Vulnerability( + package=PKG_DATA, + id="GHSA-p423-j2cm-9vmq", + aliases=["CVE-2026-39892"], + fix_versions=["46.0.7"], + description="Dummy description", +) + + class TestVulnerabilityMatcher: - def test_not_resolved(self, sample_vulnerability): + @staticmethod + def test_not_resolved(sample_vulnerability): vuln = sample_vulnerability.vulnerability matcher = VulnerabilityMatcher(current_vulnerabilities=[vuln]) assert not matcher.is_resolved(vuln) - def test_changed_id_not_resolved(self, sample_vulnerability): - vuln2 = _flip_id_and_alias(sample_vulnerability) - matcher = VulnerabilityMatcher(current_vulnerabilities=[vuln2]) + @staticmethod + def test_changed_id_not_resolved(sample_vulnerability, flipped_id_vulnerability): + """ + Simulate a vulnerability to be still present, but its ID having + changed over time. + + The test verifies that the vulnerability (using the original ID) is + still matched as "not resolved". + """ + + matcher = VulnerabilityMatcher( + current_vulnerabilities=[flipped_id_vulnerability] + ) assert not matcher.is_resolved(sample_vulnerability.vulnerability) - def test_resolved(self, sample_vulnerability): + @staticmethod + def test_resolved(sample_vulnerability): vuln = sample_vulnerability.vulnerability matcher = VulnerabilityMatcher(current_vulnerabilities=[]) assert matcher.is_resolved(vuln) + @staticmethod + @pytest.mark.parametrize( + "current_vulnerabilities", + [ + [VULN_1, VULN_2], + [VULN_2], + ], + ) + def test_no_resolution_same_package(current_vulnerabilities): + """ + Two vulnerabilities in the same package 'cryptography', none of + them resolved. + """ + + matcher = VulnerabilityMatcher(current_vulnerabilities=current_vulnerabilities) + for v in (VULN_1, VULN_2): + expected = not v in current_vulnerabilities + assert matcher.is_resolved(v) is expected + class TestDependenciesAudit: def test_no_vulnerabilities_for_previous_and_current(self): @@ -52,7 +111,7 @@ def test_vulnerability_in_current_but_not_present(self, sample_vulnerability): # only care about "resolved" vulnerabilities, not new ones assert audit.resolved_vulnerabilities == [] - def test_resolved_vulnerabilities(self, sample_vulnerability): + def test_resolved_vulnerability(self, sample_vulnerability): audit = DependenciesAudit( previous_vulnerabilities=[sample_vulnerability.vulnerability], current_vulnerabilities=[], From d16f1f30d132f1b965d6b06d6b78f9f262f407c3 Mon Sep 17 00:00:00 2001 From: ckunki Date: Mon, 20 Apr 2026 15:13:27 +0200 Subject: [PATCH 25/37] Simplified initialization of VulnerabilityMatcher._references --- .../dependencies/track_vulnerabilities.py | 21 ++++++------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/exasol/toolbox/util/dependencies/track_vulnerabilities.py b/exasol/toolbox/util/dependencies/track_vulnerabilities.py index e913a9496..8a4c1fc9e 100644 --- a/exasol/toolbox/util/dependencies/track_vulnerabilities.py +++ b/exasol/toolbox/util/dependencies/track_vulnerabilities.py @@ -1,3 +1,4 @@ +from collections import defaultdict from inspect import cleandoc from pydantic import ( @@ -12,17 +13,10 @@ class VulnerabilityMatcher: def __init__(self, current_vulnerabilities: list[Vulnerability]): # Dict of current vulnerabilities: # * keys: package names - # * values: set of each vulnerability's references - # self._references = { - # v.package.name: set(v.references) for v in current_vulnerabilities - # } - self._references = {} + # * values: set containing the union of all references of all vulnerabilities. + self._references = defaultdict(set) for v in current_vulnerabilities: - p = v.package.name - if not p in self._references: - self._references[p] = set(v.references) - entry = self._references[p] - self._references[p] = entry | set(v.references) + self._references[v.package.name] |= set(v.references) def is_resolved(self, vuln: Vulnerability) -> bool: """ @@ -69,15 +63,12 @@ def resolved_vulnerabilities(self) -> list[Vulnerability]: def report_resolved_vulnerabilities(self) -> str: if not (resolved := self.resolved_vulnerabilities): return "" - header = cleandoc( - ## Fixed Vulnerabilities - """ + header = cleandoc(""" This release fixes vulnerabilities by updating dependencies: | Dependency | Vulnerability | Affected | Fixed in | |------------|---------------|----------|----------| - """ - ) + """) def formatted(vuln: Vulnerability) -> str: columns = ( From 0866069b5497ee062ad7fea3adee18b44edf0e31 Mon Sep 17 00:00:00 2001 From: ckunki Date: Mon, 20 Apr 2026 15:15:19 +0200 Subject: [PATCH 26/37] Simplified initialization of VulnerabilityMatcher._references (2) --- exasol/toolbox/util/dependencies/track_vulnerabilities.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/exasol/toolbox/util/dependencies/track_vulnerabilities.py b/exasol/toolbox/util/dependencies/track_vulnerabilities.py index 8a4c1fc9e..dfe6ed3a5 100644 --- a/exasol/toolbox/util/dependencies/track_vulnerabilities.py +++ b/exasol/toolbox/util/dependencies/track_vulnerabilities.py @@ -11,12 +11,11 @@ class VulnerabilityMatcher: def __init__(self, current_vulnerabilities: list[Vulnerability]): - # Dict of current vulnerabilities: - # * keys: package names - # * values: set containing the union of all references of all vulnerabilities. + # Dictionary mapping package names to a unified set of all active + # vulnerability references (IDs, CVEs, aliases) for that package. self._references = defaultdict(set) for v in current_vulnerabilities: - self._references[v.package.name] |= set(v.references) + self._references[v.package.name].update(v.references) def is_resolved(self, vuln: Vulnerability) -> bool: """ From 31b6aebcdc8f9ec817620c77324937c88b0b3e35 Mon Sep 17 00:00:00 2001 From: ckunki Date: Mon, 20 Apr 2026 15:16:07 +0200 Subject: [PATCH 27/37] reverted renamed variable --- exasol/toolbox/util/dependencies/track_vulnerabilities.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exasol/toolbox/util/dependencies/track_vulnerabilities.py b/exasol/toolbox/util/dependencies/track_vulnerabilities.py index dfe6ed3a5..04f4497ab 100644 --- a/exasol/toolbox/util/dependencies/track_vulnerabilities.py +++ b/exasol/toolbox/util/dependencies/track_vulnerabilities.py @@ -34,8 +34,8 @@ def is_resolved(self, vuln: Vulnerability) -> bool: vulnerability. """ refs = set(vuln.references) - current = self._references.get(vuln.package.name, set()) - return refs.isdisjoint(current) + current_refs = self._references.get(vuln.package.name, set()) + return refs.isdisjoint(current_refs) class DependenciesAudit(BaseModel): From ae84502be60f02202a04b709e7447033ad4ab1c7 Mon Sep 17 00:00:00 2001 From: ckunki Date: Mon, 20 Apr 2026 15:24:04 +0200 Subject: [PATCH 28/37] fixed sonar finding --- exasol/toolbox/util/release/changelog.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/exasol/toolbox/util/release/changelog.py b/exasol/toolbox/util/release/changelog.py index 381f03995..4b1007f91 100644 --- a/exasol/toolbox/util/release/changelog.py +++ b/exasol/toolbox/util/release/changelog.py @@ -41,7 +41,8 @@ def __init__(self, changes_path: Path, root_path: Path, version: Version) -> Non self.version = version self.unreleased: Path = changes_path / "unreleased.md" self.versioned_changes: Path = changes_path / f"changes_{version}.md" - self.changelog: Path = changes_path / "changelog.md" + # Accepting attribute changelog duplicating the class name + self.changelog: Path = changes_path / "changelog.md" # NOSONAR self.root_path: Path = root_path def _create_new_unreleased(self): @@ -80,7 +81,7 @@ def _dependency_sections(self) -> Generator[Markdown]: def _dependency_changes(self) -> Markdown | None: if sections := list(self._dependency_sections()): - return Markdown(f"## Dependency Updates", children=sections) + return Markdown("## Dependency Updates", children=sections) return None @staticmethod From 2af7cce574716a6ed607eeb155710e7207a0a0d8 Mon Sep 17 00:00:00 2001 From: ckunki Date: Mon, 20 Apr 2026 15:54:43 +0200 Subject: [PATCH 29/37] nox -s format:fix --- exasol/toolbox/util/release/changelog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol/toolbox/util/release/changelog.py b/exasol/toolbox/util/release/changelog.py index 4b1007f91..a0b02e88c 100644 --- a/exasol/toolbox/util/release/changelog.py +++ b/exasol/toolbox/util/release/changelog.py @@ -42,7 +42,7 @@ def __init__(self, changes_path: Path, root_path: Path, version: Version) -> Non self.unreleased: Path = changes_path / "unreleased.md" self.versioned_changes: Path = changes_path / f"changes_{version}.md" # Accepting attribute changelog duplicating the class name - self.changelog: Path = changes_path / "changelog.md" # NOSONAR + self.changelog: Path = changes_path / "changelog.md" # NOSONAR self.root_path: Path = root_path def _create_new_unreleased(self): From 9a80e90851684b5949a32562e343d86867426376 Mon Sep 17 00:00:00 2001 From: Christoph Kuhnke Date: Tue, 21 Apr 2026 11:24:39 +0200 Subject: [PATCH 30/37] Apply suggestions from code review Co-authored-by: Ariel Schulz <43442541+ArBridgeman@users.noreply.github.com> --- exasol/toolbox/util/release/changelog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol/toolbox/util/release/changelog.py b/exasol/toolbox/util/release/changelog.py index a0b02e88c..fef3f880b 100644 --- a/exasol/toolbox/util/release/changelog.py +++ b/exasol/toolbox/util/release/changelog.py @@ -136,7 +136,7 @@ def _create_versioned_changes(self, initial_content: str) -> None: Create a versioned changes file. Args: - unreleased_content: the content of the (not yet versioned) changes + initial_content: the content of the (not yet versioned) changes """ versioned = Markdown.from_text(initial_content) From 9b36beb61aa22df47e976d639aa1aa612ae8cbaf Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 21 Apr 2026 11:26:41 +0200 Subject: [PATCH 31/37] Fixd review finding --- exasol/toolbox/util/release/changelog.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/exasol/toolbox/util/release/changelog.py b/exasol/toolbox/util/release/changelog.py index fef3f880b..8ce670849 100644 --- a/exasol/toolbox/util/release/changelog.py +++ b/exasol/toolbox/util/release/changelog.py @@ -129,7 +129,10 @@ def _resolved_vulnerabilities(self) -> Markdown | None: ), current_vulnerabilities=get_vulnerabilities(self.root_path), ).report_resolved_vulnerabilities() - return Markdown("## Security Issues", report) if report else None + if report: + return Markdown("## Security Issues", report) + else: + return None def _create_versioned_changes(self, initial_content: str) -> None: """ From fe3f37d3a82c83aa529d297cea8e791f6edf85ed Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 21 Apr 2026 11:31:56 +0200 Subject: [PATCH 32/37] Added Enum for sections in changes file --- exasol/toolbox/util/release/changelog.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/exasol/toolbox/util/release/changelog.py b/exasol/toolbox/util/release/changelog.py index 8ce670849..f75ed4da1 100644 --- a/exasol/toolbox/util/release/changelog.py +++ b/exasol/toolbox/util/release/changelog.py @@ -3,6 +3,7 @@ from collections import OrderedDict from collections.abc import Generator from datetime import datetime +from enum import Enum from inspect import cleandoc from pathlib import Path @@ -27,6 +28,19 @@ """) + "\n" +class Section(Enum): + FEATURES = "Features" + SECURITY = "Security Issues" + BUGFIXES = "Bugfixes" + DOCCUMENTATION = "Documentation" + REFACTORINGS = "Refactorings" + DEPENDENCIES = "Dependency Updates" + + @property + def title(self) -> str: + return f"## {self.value}" + + class Changelog: def __init__(self, changes_path: Path, root_path: Path, version: Version) -> None: """ @@ -81,7 +95,7 @@ def _dependency_sections(self) -> Generator[Markdown]: def _dependency_changes(self) -> Markdown | None: if sections := list(self._dependency_sections()): - return Markdown("## Dependency Updates", children=sections) + return Markdown(Section.DEPENDENCIES.title, children=sections) return None @staticmethod @@ -130,7 +144,7 @@ def _resolved_vulnerabilities(self) -> Markdown | None: current_vulnerabilities=get_vulnerabilities(self.root_path), ).report_resolved_vulnerabilities() if report: - return Markdown("## Security Issues", report) + return Markdown(Section.SECURITY.title, report) else: return None From bcd43bc7b572fb542cc43814151ea1a3ff60aa1b Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 21 Apr 2026 11:58:13 +0200 Subject: [PATCH 33/37] Fixed integration test --- exasol/toolbox/util/release/changelog.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/exasol/toolbox/util/release/changelog.py b/exasol/toolbox/util/release/changelog.py index f75ed4da1..bd3da6cff 100644 --- a/exasol/toolbox/util/release/changelog.py +++ b/exasol/toolbox/util/release/changelog.py @@ -137,10 +137,12 @@ def get_changed_files(self) -> list[Path]: return [self.unreleased, self.versioned_changes, self.changelog] def _resolved_vulnerabilities(self) -> Markdown | None: + try: + previous = get_vulnerabilities_from_latest_tag(self.root_path) + except LatestTagNotFoundError: + previous = [] report = DependenciesAudit( - previous_vulnerabilities=get_vulnerabilities_from_latest_tag( - self.root_path - ), + previous_vulnerabilities=previous, current_vulnerabilities=get_vulnerabilities(self.root_path), ).report_resolved_vulnerabilities() if report: From 4ddec8a29e9460d6ab49a51db441091e97b55993 Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 21 Apr 2026 14:28:19 +0200 Subject: [PATCH 34/37] fixed integration tests --- exasol/toolbox/util/dependencies/audit.py | 12 ++++-- test/integration/project-template/conftest.py | 37 ++++++++++++++++--- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/exasol/toolbox/util/dependencies/audit.py b/exasol/toolbox/util/dependencies/audit.py index 1983fc4f1..bba376e37 100644 --- a/exasol/toolbox/util/dependencies/audit.py +++ b/exasol/toolbox/util/dependencies/audit.py @@ -178,10 +178,14 @@ def audit_poetry_files(working_directory: Path) -> str: (tmpdir / requirements_txt).write_text(output.stdout) # CLI option `--disable-pip` skips dependency resolution in pip. The - # option can be used with hashed requirements files (which is the case - # here) to avoid `pip-audit` installing an isolated environment and - # speed up the audit significantly. - command = ["pip-audit", "--disable-pip", "-r", requirements_txt, "-f", "json"] + # option can be used with hashed requirements files to avoid + # `pip-audit` installing an isolated environment and speed up the + # audit significantly. + # + # In real use scenarios of the PTB we usually have hashed + # requirements. Unfortunately this is not the case for the example + # project created in the integration tests. + command = ["pip-audit", "-r", requirements_txt, "-f", "json"] output = subprocess.run( command, capture_output=True, diff --git a/test/integration/project-template/conftest.py b/test/integration/project-template/conftest.py index eadced1a5..d15190b34 100644 --- a/test/integration/project-template/conftest.py +++ b/test/integration/project-template/conftest.py @@ -1,3 +1,4 @@ +import logging import subprocess from pathlib import Path @@ -5,6 +6,8 @@ from noxconfig import PROJECT_CONFIG +LOG = logging.getLogger(__name__) + @pytest.fixture(scope="session", autouse=True) def cwd(tmp_path_factory): @@ -36,7 +39,7 @@ def new_project(cwd): capture_output=True, check=True, ) - + subprocess.run(["poetry", "lock"], check=True) return cwd / repo_name @@ -70,15 +73,37 @@ def run_command(poetry_path, git_path, new_project): """ def _run_command_fixture(command, **kwargs): + cwd = new_project + env = {"PATH": f"{Path(git_path).parent}:{Path(poetry_path).parent}"} defaults = { "capture_output": True, - "check": True, - "cwd": new_project, - "env": {"PATH": f"{Path(git_path).parent}:{Path(poetry_path).parent}"}, + "check": False, + "cwd": cwd, + "env": env, "text": True, } config = {**defaults, **kwargs} - - return subprocess.run(command, **config) + p = subprocess.run(command, **config) + if p.returncode != 0: + def text(stream) -> str: + return "" if stream is None else stream.strip() + + message = ( + f"subprocess.run() returned exit code: {p.returncode}" + f"\ncommand: {' '.join(command)}" + f"\nstdout: {text(p.stdout)}" + f"\nstderr: {text(p.stderr)}" + f"\ncwd: {cwd}" + f"\nenv: {env}" + ) + LOG.warning(message) + if kwargs.get("check", True): + raise subprocess.CalledProcessError( + p.returncode, + command, + output=p.stdout, + stderr=p.stderr, + ) + return p return _run_command_fixture From fea9426f64a932c60fb2a3d45207b62f385550c1 Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 21 Apr 2026 14:31:26 +0200 Subject: [PATCH 35/37] nox -s format:fix --- test/integration/project-template/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/integration/project-template/conftest.py b/test/integration/project-template/conftest.py index d15190b34..fd0ebb3fb 100644 --- a/test/integration/project-template/conftest.py +++ b/test/integration/project-template/conftest.py @@ -85,6 +85,7 @@ def _run_command_fixture(command, **kwargs): config = {**defaults, **kwargs} p = subprocess.run(command, **config) if p.returncode != 0: + def text(stream) -> str: return "" if stream is None else stream.strip() From 0997166f66ba5b6ac4ccca685fe6736abb3f73f2 Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 21 Apr 2026 15:16:24 +0200 Subject: [PATCH 36/37] Fixed fixture --- test/integration/project-template/conftest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/integration/project-template/conftest.py b/test/integration/project-template/conftest.py index fd0ebb3fb..cac5b96a6 100644 --- a/test/integration/project-template/conftest.py +++ b/test/integration/project-template/conftest.py @@ -77,12 +77,11 @@ def _run_command_fixture(command, **kwargs): env = {"PATH": f"{Path(git_path).parent}:{Path(poetry_path).parent}"} defaults = { "capture_output": True, - "check": False, "cwd": cwd, "env": env, "text": True, } - config = {**defaults, **kwargs} + config = {**defaults, **kwargs, "check": False} p = subprocess.run(command, **config) if p.returncode != 0: From 7e0ef6ad2c43e385705613e5d8573e985ead77e9 Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 21 Apr 2026 15:34:09 +0200 Subject: [PATCH 37/37] Added more information to PipAuditException --- exasol/toolbox/util/dependencies/audit.py | 20 ++++++++++++++----- .../{{cookiecutter.repo_name}}/pyproject.toml | 3 +++ test/integration/project-template/conftest.py | 1 - 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/exasol/toolbox/util/dependencies/audit.py b/exasol/toolbox/util/dependencies/audit.py index bba376e37..26bc4fbd3 100644 --- a/exasol/toolbox/util/dependencies/audit.py +++ b/exasol/toolbox/util/dependencies/audit.py @@ -32,13 +32,22 @@ @dataclass class PipAuditException(Exception): + command: list[str] + cwd: Path + env: dict[str, str] returncode: int stdout: str stderr: str @classmethod - def from_subprocess(cls, proc: subprocess.CompletedProcess) -> PipAuditException: - return cls(proc.returncode, proc.stdout, proc.stderr) + def from_subprocess( + cls, + proc: subprocess.CompletedProcess, + command: list[str], + cwd: Path, + env: dict[str, str] | None = None, + ) -> PipAuditException: + return cls(command, cwd, env or {}, proc.returncode, proc.stdout, proc.stderr) class VulnerabilitySource(str, Enum): @@ -164,14 +173,15 @@ def audit_poetry_files(working_directory: Path) -> str: """ requirements_txt = "requirements.txt" + command = ["poetry", "export", "--format=requirements.txt"] output = subprocess.run( - ["poetry", "export", "--format=requirements.txt"], + command, capture_output=True, text=True, cwd=working_directory, ) # nosec if output.returncode != 0: - raise PipAuditException.from_subprocess(output) + raise PipAuditException.from_subprocess(output, command, cwd=working_directory) with tempfile.TemporaryDirectory() as path: tmpdir = Path(path) @@ -199,7 +209,7 @@ def audit_poetry_files(working_directory: Path) -> str: # they both map to returncode = 1, so we have our own logic to raise errors # for the case of 2) and not 1). if not search(PIP_AUDIT_VULNERABILITY_PATTERN, output.stderr.strip()): - raise PipAuditException.from_subprocess(output) + raise PipAuditException.from_subprocess(output, command, cwd=tmpdir) return output.stdout diff --git a/project-template/{{cookiecutter.repo_name}}/pyproject.toml b/project-template/{{cookiecutter.repo_name}}/pyproject.toml index 33dcc0710..846847f94 100644 --- a/project-template/{{cookiecutter.repo_name}}/pyproject.toml +++ b/project-template/{{cookiecutter.repo_name}}/pyproject.toml @@ -33,6 +33,9 @@ include = [ "exasol/toolbox/templates/**/*" ] +[tool.poetry.requires-plugins] +poetry-plugin-export = ">=1.8" + [poetry.urls] repository = "https://github.com/exasol/{{cookiecutter.repo_name}}" homepage = "https://github.com/exasol/{{cookiecutter.repo_name}}" diff --git a/test/integration/project-template/conftest.py b/test/integration/project-template/conftest.py index cac5b96a6..979ee7e0b 100644 --- a/test/integration/project-template/conftest.py +++ b/test/integration/project-template/conftest.py @@ -39,7 +39,6 @@ def new_project(cwd): capture_output=True, check=True, ) - subprocess.run(["poetry", "lock"], check=True) return cwd / repo_name