diff --git a/.github/workflows/reusable-build-and-release-python-versions.yml b/.github/workflows/reusable-build-and-release-python-versions.yml index 4e103d2..cef12d2 100644 --- a/.github/workflows/reusable-build-and-release-python-versions.yml +++ b/.github/workflows/reusable-build-and-release-python-versions.yml @@ -45,6 +45,8 @@ jobs: fi - name: Build Python artifact using Makefile + env: + GITHUB_TOKEN: ${{ github.token }} run: | make CONTAINER_ENGINE="sudo docker" PYTHON_VERSION=${{ inputs.tag }} ARCH=${{ inputs.arch }} UBUNTU_VERSION=${{ inputs.platform-version }} diff --git a/Makefile b/Makefile index 20d8f56..5622b8e 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ endif # Versioning PYTHON_VERSION ?= 3.13.3 ACTIONS_PYTHON_VERSIONS ?= 3.15.0-alpha.5-21016111327 -POWERSHELL_VERSION ?= v7.5.2 +POWERSHELL_VERSION ?= v7.6.1 POWERSHELL_NATIVE_VERSION ?= v7.4.0 UBUNTU_VERSION ?= 24.04 TRIVY_VERSION ?= v0.69.2 @@ -152,7 +152,13 @@ verify-trivy-checksums: # 3. Build Base PowerShell Image powershell: $(PS_PREREQS) @echo "--- Building PowerShell Base Image ---" - $(Q)cd $(PS_DIR) && $(CONTAINER_ENGINE) build \ + $(Q)cd $(PS_DIR) && \ + secret_flags=""; \ + if [ -n "$${GITHUB_TOKEN:-}" ]; then \ + secret_flags="--secret id=github_token,env=GITHUB_TOKEN"; \ + fi; \ + DOCKER_BUILDKIT=1 $(CONTAINER_ENGINE) build \ + $$secret_flags \ --network=host \ --build-arg POWERSHELL_VERSION=$(POWERSHELL_VERSION) \ --build-arg POWERSHELL_NATIVE_VERSION=$(POWERSHELL_NATIVE_VERSION) \ diff --git a/PowerShell/Dockerfile b/PowerShell/Dockerfile index e8990b6..f8d3330 100644 --- a/PowerShell/Dockerfile +++ b/PowerShell/Dockerfile @@ -1,3 +1,5 @@ +# syntax=docker/dockerfile:1.6 + ARG UBUNTU_VERSION=24.04 ARG TARGETARCH ARG POWERSHELL_VERSION=v7.5.1 @@ -5,9 +7,12 @@ ARG POWERSHELL_VERSION=v7.5.1 # Base stage for reuse FROM ubuntu:${UBUNTU_VERSION} AS base ENV DEBIAN_FRONTEND=noninteractive +ENV LANG=C.UTF-8 +ENV LC_ALL=C.UTF-8 RUN apt-get -qq update && \ - apt-get -qq install -y git curl ca-certificates && \ + apt-get -qq install -y git curl ca-certificates locales && \ + locale-gen C.UTF-8 && \ apt-get clean && rm -rf /var/lib/apt/lists/* # Build libpsl-native using base @@ -48,12 +53,13 @@ COPY patch/powershell-gen-${POWERSHELL_VERSION}.tar.gz /tmp/ COPY update-dotnet-sdk-and-tfm.sh /tmp/ COPY dotnet-install.py /tmp/ -RUN chmod +x /tmp/update-dotnet-sdk-and-tfm.sh && \ +RUN --mount=type=secret,id=github_token,required=false \ + chmod +x /tmp/update-dotnet-sdk-and-tfm.sh && \ git clone https://github.com/PowerShell/PowerShell.git /PowerShell && \ cd /PowerShell && \ git checkout tags/${POWERSHELL_VERSION} -b ${TARGETARCH}-${POWERSHELL_VERSION} && \ SDK_VERSION=$(python3 -c "import json; print(json.load(open('global.json'))['sdk']['version'])") && \ - python3 /tmp/dotnet-install.py --tag $SDK_VERSION && \ + GITHUB_TOKEN_FILE=/run/secrets/github_token python3 /tmp/dotnet-install.py --tag $SDK_VERSION && \ ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet && \ git apply /tmp/powershell.patch && \ cp /tmp/update-dotnet-sdk-and-tfm.sh . && \ diff --git a/PowerShell/dotnet-install.py b/PowerShell/dotnet-install.py index fccfcbf..c6f4b0a 100644 --- a/PowerShell/dotnet-install.py +++ b/PowerShell/dotnet-install.py @@ -17,6 +17,7 @@ import urllib.error import bisect import time +from pathlib import Path from typing import Optional, List, Tuple, NamedTuple # Third-party imports @@ -29,9 +30,37 @@ NUGET_PACKAGE = "microsoft.netcore.app.runtime.linux-x64" FETCH_MAX_RETRIES = 8 FETCH_RETRY_DELAY = 5 +GITHUB_TOKEN_ENV = "GITHUB_TOKEN" +GITHUB_TOKEN_FILE_ENV = "GITHUB_TOKEN_FILE" +GITHUB_USER_AGENT = "python-versions-pz-dotnet-install" app = typer.Typer() +def get_github_token() -> str: + token = os.getenv(GITHUB_TOKEN_ENV, "").strip() + if token: + return token + + token_file = os.getenv(GITHUB_TOKEN_FILE_ENV, "").strip() + if token_file: + try: + return Path(token_file).read_text(encoding="utf-8").strip() + except Exception: + return "" + + return "" + +def build_request(url: str, accept: Optional[str] = None) -> urllib.request.Request: + headers = {"User-Agent": GITHUB_USER_AGENT} + if accept: + headers["Accept"] = accept + + token = get_github_token() + if token: + headers["Authorization"] = f"Bearer {token}" + + return urllib.request.Request(url, headers=headers) + def get_nuget_versions(package: str) -> List[str]: """Fetch official .NET runtime versions from NuGet.org.""" url = f"https://api.nuget.org/v3-flatcontainer/{package}/index.json" @@ -179,7 +208,8 @@ def fetch_json(url: str) -> List[dict]: """Download and parse JSON response from a given URL with basic retries.""" for attempt in range(FETCH_MAX_RETRIES): try: - with urllib.request.urlopen(url) as response: + request = build_request(url, "application/vnd.github+json") + with urllib.request.urlopen(request) as response: if response.status >= 400: raise typer.Exit(f"❌ Failed to fetch {url}") return json.loads(response.read()) @@ -257,7 +287,8 @@ def select_tag_interactive(tags: List[dict], filter_prefix: Optional[str]) -> st def download_file(url: str, dest_path: str) -> None: """Download a file from a URL to a destination path.""" - with urllib.request.urlopen(url) as response: + request = build_request(url) + with urllib.request.urlopen(request) as response: if response.status >= 400: raise typer.Exit(f"❌ Failed to download {url}") if "html" in response.headers.get("Content-Type", "").lower(): diff --git a/PowerShell/patch/powershell-gen-v7.6.1.tar.gz b/PowerShell/patch/powershell-gen-v7.6.1.tar.gz new file mode 100644 index 0000000..61ef52a Binary files /dev/null and b/PowerShell/patch/powershell-gen-v7.6.1.tar.gz differ diff --git a/PowerShell/patch/powershell-ppc64le-v7.6.1.patch b/PowerShell/patch/powershell-ppc64le-v7.6.1.patch new file mode 100644 index 0000000..a709961 --- /dev/null +++ b/PowerShell/patch/powershell-ppc64le-v7.6.1.patch @@ -0,0 +1,90 @@ +diff --git a/build.psm1 b/build.psm1 +index 42a202d94..0ca7cf926 100644 +--- a/build.psm1 ++++ b/build.psm1 +@@ -425,6 +425,7 @@ function Start-PSBuild { + "linux-arm", + "linux-arm64", + "linux-x64", ++ "linux-ppc64le", + "osx-arm64", + "osx-x64", + "win-arm", +@@ -612,6 +613,16 @@ Fix steps: + } + + $Arguments += "/property:AppDeployment=$AppDeployment" ++ ++ # As the ReadyToRun package for linux-ppc64le is not available on NuGet, ++ # we must explicitly disable ReadyToRun compilation for this runtime. ++ # This addresses the NETSDK1094 error for this specific platform. ++ if ($Options.Runtime -eq 'linux-ppc64le') { ++ $Arguments += "/property:PublishReadyToRun=false" ++ $Arguments += "/property:WarnAsError=false" # Do not treat warnings as errors for ppc64le ++ $Arguments += "/property:RunAnalyzers=false" # Disable analyzers for ppc64le ++ } ++ + $Arguments += "--configuration", $Options.Configuration + $Arguments += "--framework", $Options.Framework + +@@ -1220,6 +1231,7 @@ function New-PSOptions { + "linux-arm", + "linux-arm64", + "linux-x64", ++ "linux-ppc64le", + "osx-arm64", + "osx-x64", + "win-arm", +@@ -4691,6 +4703,9 @@ function Clear-NativeDependencies + '.*-arm64' { + $diasymFileName = $diasymFileNamePattern -f 'arm64' + } ++ '.*-ppc64le' { ++ $diasymFileName = $diasymFileNamePattern -f 'ppc64le' ++ } + 'fxdependent.*' { + Write-Verbose -Message "$($script:Options.Runtime) is a fxdependent runtime, no cleanup needed in pwsh.deps.json" -Verbose + return +@@ -4790,3 +4805,4 @@ function Set-PipelineVariable { + # also set in the current session + Set-Item -Path "env:$Name" -Value $Value + } ++ +diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 +index b0131d39e..5adab011c 100644 +--- a/tools/packaging/packaging.psm1 ++++ b/tools/packaging/packaging.psm1 +@@ -624,6 +624,15 @@ function Start-PSPackage { + } + } + 'deb' { ++ # Determine the host architecture dynamically ++ $hostArchitecture = switch ($Runtime) { ++ 'linux-arm64' { 'arm64' } ++ 'linux-arm' { 'armhf' } # Assuming arm32 is armhf for Debian ++ 'linux-ppc64le' { 'ppc64el' } # Debian uses 'ppc64el' for ppc64le ++ 'linux-x64' { 'amd64' } ++ default { throw "Unsupported runtime architecture: $Runtime" } ++ } ++ + $Arguments = @{ + Type = 'deb' + PackageSourcePath = $Source +@@ -632,7 +641,7 @@ function Start-PSPackage { + Force = $Force + NoSudo = $NoSudo + LTS = $LTS +- HostArchitecture = "amd64" ++ HostArchitecture = $hostArchitecture + } + foreach ($Distro in $Script:DebianDistributions) { + $Arguments["Distribution"] = $Distro +@@ -1064,7 +1073,7 @@ function New-UnixPackage { + # Host architecture values allowed for rpm type packages include: x86_64, aarch64, native, all, noarch, any + # Host architecture values allowed for osxpkg type packages include: x86_64, arm64 + [string] +- [ValidateSet("x86_64", "amd64", "aarch64", "arm64", "native", "all", "noarch", "any")] ++ [ValidateSet("x86_64", "amd64", "aarch64", "arm64", "ppc64el", "native", "all", "noarch", "any")] + $HostArchitecture, + + [Switch] diff --git a/PowerShell/patch/powershell-s390x-v7.6.1.patch b/PowerShell/patch/powershell-s390x-v7.6.1.patch new file mode 100644 index 0000000..df1e440 --- /dev/null +++ b/PowerShell/patch/powershell-s390x-v7.6.1.patch @@ -0,0 +1,90 @@ +diff --git a/build.psm1 b/build.psm1 +index 42a202d94..428a357be 100644 +--- a/build.psm1 ++++ b/build.psm1 +@@ -425,6 +425,7 @@ function Start-PSBuild { + "linux-arm", + "linux-arm64", + "linux-x64", ++ "linux-s390x", + "osx-arm64", + "osx-x64", + "win-arm", +@@ -612,6 +613,16 @@ Fix steps: + } + + $Arguments += "/property:AppDeployment=$AppDeployment" ++ ++ # As the ReadyToRun package for linux-s390x is not available on NuGet, ++ # we must explicitly disable ReadyToRun compilation for this runtime. ++ # This addresses the NETSDK1094 error for this specific platform. ++ if ($Options.Runtime -eq 'linux-s390x') { ++ $Arguments += "/property:PublishReadyToRun=false" ++ $Arguments += "/property:WarnAsError=false" # Do not treat warnings as errors for s390x ++ $Arguments += "/property:RunAnalyzers=false" # Disable analyzers for s390x ++ } ++ + $Arguments += "--configuration", $Options.Configuration + $Arguments += "--framework", $Options.Framework + +@@ -1220,6 +1231,7 @@ function New-PSOptions { + "linux-arm", + "linux-arm64", + "linux-x64", ++ "linux-s390x", + "osx-arm64", + "osx-x64", + "win-arm", +@@ -4691,6 +4703,9 @@ function Clear-NativeDependencies + '.*-arm64' { + $diasymFileName = $diasymFileNamePattern -f 'arm64' + } ++ '.*-s390x' { ++ $diasymFileName = $diasymFileNamePattern -f 's390x' ++ } + 'fxdependent.*' { + Write-Verbose -Message "$($script:Options.Runtime) is a fxdependent runtime, no cleanup needed in pwsh.deps.json" -Verbose + return +@@ -4790,3 +4805,4 @@ function Set-PipelineVariable { + # also set in the current session + Set-Item -Path "env:$Name" -Value $Value + } ++ +diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 +index b0131d39e..327dc5305 100644 +--- a/tools/packaging/packaging.psm1 ++++ b/tools/packaging/packaging.psm1 +@@ -624,6 +624,15 @@ function Start-PSPackage { + } + } + 'deb' { ++ # Determine the host architecture dynamically ++ $hostArchitecture = switch ($Runtime) { ++ 'linux-arm64' { 'arm64' } ++ 'linux-arm' { 'armhf' } # Assuming arm32 is armhf for Debian ++ 'linux-s390x' { 's390x' } ++ 'linux-x64' { 'amd64' } ++ default { throw "Unsupported runtime architecture: $Runtime" } ++ } ++ + $Arguments = @{ + Type = 'deb' + PackageSourcePath = $Source +@@ -632,7 +641,7 @@ function Start-PSPackage { + Force = $Force + NoSudo = $NoSudo + LTS = $LTS +- HostArchitecture = "amd64" ++ HostArchitecture = $hostArchitecture + } + foreach ($Distro in $Script:DebianDistributions) { + $Arguments["Distribution"] = $Distro +@@ -1064,7 +1073,7 @@ function New-UnixPackage { + # Host architecture values allowed for rpm type packages include: x86_64, aarch64, native, all, noarch, any + # Host architecture values allowed for osxpkg type packages include: x86_64, arm64 + [string] +- [ValidateSet("x86_64", "amd64", "aarch64", "arm64", "native", "all", "noarch", "any")] ++ [ValidateSet("x86_64", "amd64", "aarch64", "arm64", "s390x", "native", "all", "noarch", "any")] + $HostArchitecture, + + [Switch] diff --git a/tests/test_dotnet_install.py b/tests/test_dotnet_install.py index 33caf09..955daca 100644 --- a/tests/test_dotnet_install.py +++ b/tests/test_dotnet_install.py @@ -299,6 +299,24 @@ def test_download_file_success(self, mock_copy, mock_urlopen, temp_file): download_file("https://example.com/file.tar.gz", temp_file) mock_copy.assert_called_once() + @patch('urllib.request.urlopen') + @patch('shutil.copyfileobj') + def test_download_file_uses_github_token_file(self, mock_copy, mock_urlopen, temp_file, tmp_path): + """Test download_file sends the GitHub token from a file when available.""" + token_file = tmp_path / "token.txt" + token_file.write_text("ghs_file_token\n", encoding="utf-8") + + mock_response = MagicMock() + mock_response.status = 200 + mock_response.headers = {"Content-Type": "application/gzip"} + mock_urlopen.return_value.__enter__.return_value = mock_response + + with patch.dict(os.environ, {"GITHUB_TOKEN_FILE": str(token_file)}, clear=False): + download_file("https://example.com/file.tar.gz", temp_file) + + request = mock_urlopen.call_args[0][0] + assert request.get_header("Authorization") == "Bearer ghs_file_token" + @patch('urllib.request.urlopen') def test_download_file_http_error(self, mock_urlopen): """Test download with HTTP error.""" @@ -372,6 +390,20 @@ def test_fetch_json_success(self, mock_urlopen): result = fetch_json("https://api.github.com/repos/test/test/releases") assert result == test_data + @patch.dict(os.environ, {"GITHUB_TOKEN": "ghs_test_token"}, clear=False) + @patch('urllib.request.urlopen') + def test_fetch_json_uses_github_token(self, mock_urlopen): + """Test fetch_json sends the GitHub token when it is available.""" + mock_response = MagicMock() + mock_response.status = 200 + mock_response.read.return_value = json.dumps([]).encode() + mock_urlopen.return_value.__enter__.return_value = mock_response + + fetch_json("https://api.github.com/repos/test/test/releases") + + request = mock_urlopen.call_args[0][0] + assert request.get_header("Authorization") == "Bearer ghs_test_token" + @patch('urllib.request.urlopen') def test_fetch_json_http_error(self, mock_urlopen): """Test fetch with HTTP error."""