diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index acbbd7f..9ea7dab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,7 @@ jobs: "nginx", "node", "playwright-deps", + "python", "sonar-scanner-cli", "system-packages", "timezone", diff --git a/README.md b/README.md index b333188..3862f4f 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ Below is a list with included features, click on the link for more details. | [nginx](./features/src/nginx/README.md) | A package which installs Nginx. | | [node](./features/src/node/README.md) | A package which installs Node.js. | | [playwright-deps](./features/src/playwright-deps/README.md) | A package which installs the needed dependencies to run Playwright. | +| [python](./features/src/python/README.md) | A package which installs Python. | | [sonar-scanner-cli](./features/src/sonar-scanner-cli/README.md) | A package which installs the SonarScanner CLI. | | [system-packages](./features/src/system-packages/README.md) | Install arbitrary system packages using apt or apk. | | [timezone](./features/src/timezone/README.md) | A package which allows setting the timezone. | diff --git a/build/build.go b/build/build.go index 5ab9649..ab35c6c 100644 --- a/build/build.go +++ b/build/build.go @@ -267,6 +267,17 @@ func init() { return publishFeature("playwright-deps") }) + ////////// python + gotaskr.Task("Feature:python:Package", func() error { + return packageFeature("python") + }) + gotaskr.Task("Feature:python:Test", func() error { + return testFeature("python") + }) + gotaskr.Task("Feature:python:Publish", func() error { + return publishFeature("python") + }) + ////////// sonar-scanner-cli gotaskr.Task("Feature:sonar-scanner-cli:Package", func() error { return packageFeature("sonar-scanner-cli") diff --git a/features/src/python/README.md b/features/src/python/README.md new file mode 100755 index 0000000..6eb8971 --- /dev/null +++ b/features/src/python/README.md @@ -0,0 +1,34 @@ +# Python (python) + +A package which installs Python. + +## Example Usage + +```json +"features": { + "ghcr.io/postfinance/devcontainer-features/python:0.1.0": { + "version": "latest", + "downloadUrl": "", + "pipIndex": "", + "pipIndexUrl": "", + "pipTrustedHost": "" + } +} +``` + +## Options + +| Option | Description | Type | Default Value | Proposals | +|-----|-----|-----|-----|-----| +| version | The version of Python to install. | string | latest | latest, 3.12, 3.9.19 | +| downloadUrl | The download URL to use. | string | <empty> | https://mycompany.com/artifactory/python-generic-ftp-remote, https://mycompany.com/artifactory/python-generic-remote/ftp/python | +| pipIndex | The pip index to use (used by search). | string | <empty> | https://mycompany.com/artifactory/api/pypi/python/simple, https://mycompany.com/nexus/repository/pypi-group/pypi | +| pipIndexUrl | The pip index URL to use (used by install). | string | <empty> | https://mycompany.com/artifactory/api/pypi/python/simple, https://mycompany.com/nexus/repository/pypi-group/simple | +| pipTrustedHost | The pip trusted host to use. | string | <empty> | mycompany.com, artifactory.mycompany.com, nexus.mycompany.com | + +## Customizations + +### VS Code Extensions + +- `ms-python.python` +- `ms-python.vscode-pylance` diff --git a/features/src/python/devcontainer-feature.json b/features/src/python/devcontainer-feature.json new file mode 100644 index 0000000..1536e3a --- /dev/null +++ b/features/src/python/devcontainer-feature.json @@ -0,0 +1,70 @@ +{ + "id": "python", + "version": "0.1.0", + "name": "Python", + "description": "A package which installs Python.", + "options": { + "version": { + "type": "string", + "proposals": [ + "latest", + "3.12", + "3.9.19" + ], + "default": "latest", + "description": "The version of Python to install." + }, + "downloadUrl": { + "type": "string", + "default": "", + "proposals": [ + "https://mycompany.com/artifactory/python-generic-ftp-remote", + "https://mycompany.com/artifactory/python-generic-remote/ftp/python" + ], + "description": "The download URL to use." + }, + "pipIndex": { + "type": "string", + "default": "", + "proposals": [ + "https://mycompany.com/artifactory/api/pypi/python/simple", + "https://mycompany.com/nexus/repository/pypi-group/pypi" + ], + "description": "The pip index to use (used by search)." + }, + "pipIndexUrl": { + "type": "string", + "default": "", + "proposals": [ + "https://mycompany.com/artifactory/api/pypi/python/simple", + "https://mycompany.com/nexus/repository/pypi-group/simple" + ], + "description": "The pip index URL to use (used by install)." + }, + "pipTrustedHost": { + "type": "string", + "default": "", + "proposals": [ + "mycompany.com", + "artifactory.mycompany.com", + "nexus.mycompany.com" + ], + "description": "The pip trusted host to use." + } + }, + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance" + ], + "settings": { + "python.defaultInterpreterPath": "/usr/local/python/current/bin/python" + } + } + }, + "containerEnv": { + "PATH": "/usr/local/python/current/bin:${PATH}", + "PYTHON_PATH": "/usr/local/python/current" + } +} \ No newline at end of file diff --git a/features/src/python/install.sh b/features/src/python/install.sh new file mode 100755 index 0000000..0ac43b8 --- /dev/null +++ b/features/src/python/install.sh @@ -0,0 +1,8 @@ +. ./functions.sh + +"./installer_$(detect_arch)" \ + -version="${VERSION:-"latest"}" \ + -downloadUrl="${DOWNLOADURL:-""}" \ + -pipIndex="${PIP_INDEX:-""}" \ + -pipIndexUrl="${PIP_INDEX_URL:-""}" \ + -pipTrustedHost="${PIP_TRUSTED_HOST:-""}" diff --git a/features/src/python/installer.go b/features/src/python/installer.go new file mode 100644 index 0000000..84c9c30 --- /dev/null +++ b/features/src/python/installer.go @@ -0,0 +1,248 @@ +package main + +import ( + "builder/installer" + "flag" + "fmt" + "os" + "path/filepath" + "regexp" + "runtime" + + "github.com/roemer/goext" + "github.com/roemer/gover" +) + +////////// +// Configuration +////////// + +var pythonVersionRegexp *regexp.Regexp = regexp.MustCompile(`^(?P\d+)\.(?P\d+)(?:\.(?P\d+))?(?:(?P[a-z]+)(?P\d+))?$`) + +////////// +// Main +////////// + +func main() { + if err := runMain(); err != nil { + fmt.Printf("Error: %v\n", err) + os.Exit(1) + } +} + +func runMain() error { + // Handle the flags + version := flag.String("version", "lts", "") + downloadUrl := flag.String("downloadUrl", "", "") + pipIndex := flag.String("pipIndex", "", "") + pipIndexUrl := flag.String("pipIndexUrl", "", "") + pipTrustedHost := flag.String("pipTrustedHost", "", "") + flag.Parse() + + // Load settings from an external file + if err := installer.LoadOverrides(); err != nil { + return err + } + + installer.HandleOverride(downloadUrl, "https://www.python.org/ftp/python", "python-download-url") + installer.HandleOverride(pipIndex, "", "python-pip-index") + installer.HandleOverride(pipIndexUrl, "", "python-pip-index-url") + installer.HandleOverride(pipTrustedHost, "", "python-pip-trusted-host") + + // Create and process the feature + feature := installer.NewFeature("Python", false, + &pythonComponent{ + ComponentBase: installer.NewComponentBase("Python", *version), + DownloadUrl: *downloadUrl, + PipIndex: *pipIndex, + PipIndexUrl: *pipIndexUrl, + PipTrustedHost: *pipTrustedHost, + }, + ) + return feature.Process() +} + +////////// +// Implementation +////////// + +type pythonComponent struct { + *installer.ComponentBase + DownloadUrl string + PipIndex string + PipIndexUrl string + PipTrustedHost string +} + +func (c *pythonComponent) GetAllVersions() ([]*gover.Version, error) { + allTags, err := installer.Tools.GitHub.GetTags("python", "cpython") + if err != nil { + return nil, err + } + return installer.Tools.Versioning.ParseVersionsFromList(allTags, pythonVersionRegexp, true) +} + +func (c *pythonComponent) InstallVersion(version *gover.Version) error { + // Download the file + name := fmt.Sprintf("Python-%s", version.Raw) + fileName := fmt.Sprintf("%s.tgz", name) + // The folder only contains the major, minor and sometimes the patch version + majorMinorName := fmt.Sprintf("%d.%d", version.Major(), version.Minor()) + folderName := majorMinorName + if version.Segments[2].IsDefined() { + folderName += fmt.Sprintf(".%d", version.Segments[2].Number) + } + downloadUrl, err := installer.Tools.Http.BuildUrl(c.DownloadUrl, folderName, fileName) + if err != nil { + return err + } + if err := installer.Tools.Download.ToFile(downloadUrl, fileName, "Python"); err != nil { + return err + } + // Extract it + extractDir := "./python-src" + if err := installer.Tools.Compression.ExtractTarGz(fileName, extractDir, false); err != nil { + return err + } + // Build + if err := c.buildPython(filepath.Join(extractDir, name), version); err != nil { + return err + } + // Create a symlink to the installed python version + symLinkPath := "/usr/local/python/current" + targetPath := fmt.Sprintf("/usr/local/python/%s", folderName) + if err := installer.Tools.FileSystem.CreateSymLink(targetPath, symLinkPath, false); err != nil { + return err + } + // Configure Pip + pythonPath := filepath.Join(targetPath, "bin", fmt.Sprintf("%s%s", "python", majorMinorName)) + if err := c.configurePip(pythonPath); err != nil { + return err + } + // Create comfortable symlink for executables + symLinkFiles := []string{ + "python", "pip", "pydoc", "idle", "python-config", + } + for _, symLinkFile := range symLinkFiles { + symTargetPath := filepath.Join(targetPath, "bin", fmt.Sprintf("%s%s", symLinkFile, majorMinorName)) + symPath := filepath.Join(targetPath, "bin", symLinkFile) + if err := installer.Tools.FileSystem.CreateSymLink(symTargetPath, symPath, false); err != nil { + return fmt.Errorf("failed creating symlink from '%s' to '%s': %w", symPath, symTargetPath, err) + } + } + // Cleanup + if err := os.RemoveAll(extractDir); err != nil { + return err + } + if err := os.Remove(fileName); err != nil { + return err + } + + // Other things? + //https://github.com/devcontainers/features/blob/main/src/python/install.sh + + return nil +} + +func (c *pythonComponent) buildPython(extractDir string, version *gover.Version) error { + // Install dependencies + if err := c.installBuildDependencies(); err != nil { + return err + } + // Create the command runner + cmdRunner := goext.CmdRunners.Console.WithWorkingDirectory(extractDir) + // Configure + if err := cmdRunner.Run("./configure", fmt.Sprintf("--prefix=/usr/local/python/%s/", version.Raw)); err != nil { + return err + } + // Build + if err := cmdRunner.Run("make", "-s", fmt.Sprintf("-j%d", runtime.NumCPU())); err != nil { + return err + } + // Install + if err := cmdRunner.Run("make", "install"); err != nil { + return err + } + return nil +} + +func (c *pythonComponent) configurePip(pythonPath string) error { + // Ensure pip is installed + if err := goext.CmdRunners.Console.Run(pythonPath, goext.Cmd.SplitArgs("-m ensurepip")...); err != nil { + return err + } + // Configure pip + if c.PipIndex != "" { + if err := goext.CmdRunners.Console.Run(pythonPath, goext.Cmd.SplitArgs("-m pip config --global set global.index", c.PipIndex)...); err != nil { + return err + } + } + if c.PipIndexUrl != "" { + if err := goext.CmdRunners.Console.Run(pythonPath, goext.Cmd.SplitArgs("-m pip config --global set global.index-url", c.PipIndexUrl)...); err != nil { + return err + } + } + if c.PipTrustedHost != "" { + if err := goext.CmdRunners.Console.Run(pythonPath, goext.Cmd.SplitArgs("-m pip config --global set global.trusted-host", c.PipTrustedHost)...); err != nil { + return err + } + } + return nil +} + +func (c *pythonComponent) installBuildDependencies() error { + return installer.Tools.System.InstallPackagesByOs(func(osInfo *installer.OsInfo) ([]string, error) { + if osInfo.IsDebian() || osInfo.IsUbuntu() { + return []string{ + "build-essential", + "gdb", + "lcov", + "pkg-config", + "libbz2-dev", + "libffi-dev", + "libgdbm-dev", + "libgdbm-compat-dev", + "liblzma-dev", + "libncurses5-dev", + "libreadline-dev", + "libsqlite3-dev", + "libssl-dev", + "tk-dev", + "uuid-dev", + "zlib1g-dev", + }, nil + } else if osInfo.IsAlpine() { + return []string{ + "bluez-dev", + "bzip2-dev", + "dpkg-dev", + "dpkg", + "findutils", + "gcc", + "gdbm-dev", + "gnupg", + "libc-dev", + "libffi-dev", + "libnsl-dev", + "libtirpc-dev", + "linux-headers", + "make", + "ncurses-dev", + "openssl-dev", + "pax-utils", + "readline-dev", + "sqlite-dev", + "tar", + "tcl-dev", + "tk", + "tk-dev", + "util-linux-dev", + "xz", + "xz-dev", + "zlib-dev", + "zstd-dev", + }, nil + } + return nil, fmt.Errorf("unsupported OS vendor: %s", osInfo.Vendor) + }) +} diff --git a/features/src/system-packages/installer.go b/features/src/system-packages/installer.go index 0c226ac..753d33c 100644 --- a/features/src/system-packages/installer.go +++ b/features/src/system-packages/installer.go @@ -41,7 +41,7 @@ type systemPackagesComponent struct { } func (c *systemPackagesComponent) InstallVersion(version *gover.Version) error { - return installer.Tools.System.InstallPackages(c.Packages) + return installer.Tools.System.InstallPackages(c.Packages...) } func parsePackages(flagValue string) []string { diff --git a/features/test/python/install.sh b/features/test/python/install.sh new file mode 100755 index 0000000..1d63bf6 --- /dev/null +++ b/features/test/python/install.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -e + +[[ -f "$(dirname "$0")/../functions.sh" ]] && source "$(dirname "$0")/../functions.sh" +[[ -f "$(dirname "$0")/functions.sh" ]] && source "$(dirname "$0")/functions.sh" + +check_version "$(python --version)" "Python 3.9.19" diff --git a/features/test/python/scenarios.json b/features/test/python/scenarios.json new file mode 100644 index 0000000..f3bf969 --- /dev/null +++ b/features/test/python/scenarios.json @@ -0,0 +1,15 @@ +{ + "install": { + "build": { + "dockerfile": "Dockerfile", + "options": [ + "--add-host=host.docker.internal:host-gateway" + ] + }, + "features": { + "./python": { + "version": "3.9.19" + } + } + } +} \ No newline at end of file diff --git a/features/test/python/test-images.json b/features/test/python/test-images.json new file mode 100644 index 0000000..f4cf196 --- /dev/null +++ b/features/test/python/test-images.json @@ -0,0 +1,6 @@ +[ + "mcr.microsoft.com/devcontainers/base:debian-11", + "mcr.microsoft.com/devcontainers/base:debian-12", + "mcr.microsoft.com/devcontainers/base:alpine", + "mcr.microsoft.com/devcontainers/base:ubuntu-24.04" +] \ No newline at end of file diff --git a/go.mod b/go.mod index 1545a4f..7aa7314 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module builder go 1.24.5 require ( + github.com/roemer/goext v0.8.1 github.com/roemer/gotaskr v0.6.0 github.com/roemer/gover v0.8.0 github.com/schollz/progressbar/v3 v3.18.0 diff --git a/go.sum b/go.sum index a2f06f8..0f1fc17 100644 --- a/go.sum +++ b/go.sum @@ -108,6 +108,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/roemer/goext v0.8.1 h1:947ssu/PoE/NfRogaOuYf/foTmlGuPOZJYrpY2R4SP0= +github.com/roemer/goext v0.8.1/go.mod h1:D+HvjseqdlevkQ9QeCME6XvXoUfJwmlyRrY3YfmTnB8= github.com/roemer/gotaskr v0.6.0 h1:TfQbMfiVGKsfXWOBf33h6gcc3bknhNnEip6YgLhLJ4w= github.com/roemer/gotaskr v0.6.0/go.mod h1:ELbNvPC6EMI+gySWiAPK0wQgOkXlnGbYSzqHvWvTHh8= github.com/roemer/gover v0.8.0 h1:DbzdgftIhofqW9M9zHSAmg5aZpabhuoM8H5NeQfM5Kk= diff --git a/installer/apt.go b/installer/apt.go index d3944d6..5c177e6 100644 --- a/installer/apt.go +++ b/installer/apt.go @@ -11,15 +11,16 @@ import ( type apt struct{} func (a apt) InstallDependencies(dependencies ...string) error { + if len(dependencies) == 0 { + return nil + } if err := execr.Run(false, "apt-get", "update"); err != nil { return err } - args := append([]string{"install", "-y"}, dependencies...) if err := execr.Run(true, "apt-get", args...); err != nil { return err } - a.CleanCache() return nil } diff --git a/installer/system.go b/installer/system.go index 37f8397..d749da8 100644 --- a/installer/system.go +++ b/installer/system.go @@ -15,11 +15,15 @@ const ( type system struct{} -func (s *system) InstallPackages(packages []string) error { +func (s *system) InstallPackages(packages ...string) error { osInfo, err := s.GetOsInfo() if err != nil { return err } + return s.InstallPackagesForOs(osInfo, packages...) +} + +func (s *system) InstallPackagesForOs(osInfo *OsInfo, packages ...string) error { switch { case osInfo.IsDebian(), osInfo.IsUbuntu(): return Tools.Apt.InstallDependencies(packages...) @@ -30,6 +34,21 @@ func (s *system) InstallPackages(packages []string) error { } } +func (s *system) InstallPackagesByOs(f func(osInfo *OsInfo) ([]string, error)) error { + osInfo, err := s.GetOsInfo() + if err != nil { + return err + } + packages, err := f(osInfo) + if err != nil { + return err + } + if packages == nil { + return nil + } + return s.InstallPackagesForOs(osInfo, packages...) +} + func (s *system) MapArchitecture(mapping map[string]string) (string, error) { mappedValue, ok := mapping[runtime.GOARCH] if !ok { diff --git a/override-all.env b/override-all.env index 29f0176..ff76a3e 100644 --- a/override-all.env +++ b/override-all.env @@ -48,6 +48,12 @@ NGINX_DOWNLOAD_URL="" NODE_DOWNLOAD_URL="" NODE_VERSIONS_URL="" +# python +PYTHON_DOWNLOAD_URL="" +PYTHON_PIP_INDEX="" +PYTHON_PIP_INDEX_URL="" +PYTHON_PIP_TRUSTED_HOST="" + # vault-cli VAULT_CLI_DOWNLOAD_URL="" VAULT_CLI_VERSIONS_URL=""