PipelineManager 0.1: gemeinsame Docker-Werkzeuge und Strato-Deployment-Vertrag
PipelineManager Image / image (push) Successful in 2m44s

This commit is contained in:
Lars Stommer committed 2026-10-26 13:15:34 +01:00
commit a6e159d125
12 files changed
+807

No files matched your search

+300
View File
@@ -0,0 +1,300 @@
"""Pipeline contract v1. Python standard library only; applications supply TOML and argv hooks."""
import argparse
import hashlib
import json
import os
from pathlib import Path, PurePosixPath
import re
import shlex
import subprocess
import tarfile
import tempfile
import time
import tomllib
import urllib.request
VERSION = "0.1.0"
NAME = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_.-]*$")
COMMIT = re.compile(r"^[0-9a-f]{40}$")
class PipelineError(Exception):
pass
def relative_path(value):
if not isinstance(value, str) or not value or "\\" in value:
raise PipelineError("Ungültiger relativer Pfad.")
path = PurePosixPath(value)
if path.is_absolute() or any(part in ("..", ".") for part in value.split("/")):
raise PipelineError("Absolute Pfade und Pfadwechsel sind nicht erlaubt.")
return value
def commands(value):
if not isinstance(value, list) or any(
not isinstance(command, list) or not command
or any(not isinstance(arg, str) or not arg or "\x00" in arg for arg in command)
for command in value
):
raise PipelineError("Befehle müssen Listen von Argumentlisten sein.")
return value
def load_config(path):
with open(path, "rb") as source:
config = tomllib.load(source)
if config.get("schema") != 1 or not NAME.fullmatch(config.get("project", "")):
raise PipelineError("Projektname oder Konfigurationsversion ungültig.")
if config.get("manager") != VERSION:
raise PipelineError("Dieses Projekt benötigt eine andere Manager-Version.")
build = config.get("build", {})
for phase in ("prepare", "test", "finalize"):
commands(build.get(phase, []))
if not build.get("test"):
raise PipelineError("Mindestens ein Testbefehl ist erforderlich.")
for item in build.get("exclude", []):
relative_path(item)
for item in build.get("required", []):
relative_path(item)
for target in config.get("targets", {}).values():
if not isinstance(target.get("enabled", False), bool):
raise PipelineError("enabled muss true oder false sein.")
directory = target.get("path", "")
if not directory or any(not NAME.fullmatch(part) or part in (".", "..") for part in directory.split("/")):
raise PipelineError("Unsicheres Zielverzeichnis.")
if not target.get("url", "").startswith("https://"):
raise PipelineError("Gesundheitscheck benötigt HTTPS.")
remote = config.get("remote", {})
relative_path(remote.get("public", "public"))
for destination, source in remote.get("shared", {}).items():
relative_path(destination)
relative_path(source)
commands(remote.get("backup", []))
commands(remote.get("activate", []))
return config
def execute(argv, cwd=None, env=None, **options):
return subprocess.run(argv, cwd=cwd, env=env, check=True, **options)
def git(project, *arguments):
return execute(["git", "-c", f"safe.directory={Path(project).resolve()}", "-C", str(project), *arguments], capture_output=True, text=True).stdout.strip()
def digest(path):
with open(path, "rb") as source:
return hashlib.file_digest(source, "sha256").hexdigest()
def inspect_package(archive, config, revision):
checksum = Path(str(archive) + ".sha256")
expected = checksum.read_text().strip()
if not re.fullmatch(r"[0-9a-f]{64}", expected) or digest(archive) != expected:
raise PipelineError("Paket-Prüfsumme stimmt nicht.")
with tarfile.open(archive, "r:gz") as package:
names = set()
for member in package.getmembers():
name = relative_path(member.name)
if member.issym() or member.islnk() or not (member.isfile() or member.isdir()):
raise PipelineError("Links und Spezialdateien sind im Paket nicht erlaubt.")
if any(part == ".git" or part == ".env" or part.startswith(".env.") for part in PurePosixPath(name).parts):
raise PipelineError("Geheimnisdatei im Paket.")
if name in names:
raise PipelineError("Doppelter Paketpfad.")
names.add(name)
metadata = json.load(package.extractfile("PIPELINE.json"))
if metadata != {"schema": 1, "project": config["project"], "revision": revision, "tests": "passed"}:
raise PipelineError("Paket gehört nicht zum geprüften Projekt und Commit.")
if not set(config["build"].get("required", [])).issubset(names):
raise PipelineError("Pflichtdateien fehlen im Paket.")
return expected
def prepare(project, config, output):
project = Path(project).resolve()
revision = git(project, "rev-parse", "HEAD")
if not COMMIT.fullmatch(revision):
raise PipelineError("Ungültige Git-Revision.")
output = Path(output).resolve()
output.parent.mkdir(parents=True, exist_ok=True)
if output.exists():
raise PipelineError("Ausgabepaket existiert bereits.")
with tempfile.TemporaryDirectory(prefix="pipeline-build-") as temp:
work = Path(temp)
export = work / "source.tar"
execute(["git", "-C", str(project), "archive", "--output", str(export), revision])
source = work / "source"
source.mkdir()
with tarfile.open(export) as package:
for member in package.getmembers():
relative_path(member.name.rstrip("/"))
if member.issym() or member.islnk() or not (member.isfile() or member.isdir()):
raise PipelineError("Git-Export enthält einen Link oder eine Spezialdatei.")
package.extractall(source)
environment = os.environ.copy()
# Deployment secrets are supplied only to the later deployment step.
for key in list(environment):
if key.startswith(("DEPLOY_", "SSH_")):
del environment[key]
environment.update({str(key): str(value) for key, value in config["build"].get("environment", {}).items()})
for phase in ("prepare", "test", "finalize"):
print(f"Phase: {phase}", flush=True)
for command in config["build"].get(phase, []):
execute(command, cwd=source, env=environment)
metadata = {"schema": 1, "project": config["project"], "revision": revision, "tests": "passed"}
(source / "PIPELINE.json").write_text(json.dumps(metadata), encoding="utf-8")
excluded = [PurePosixPath(value) for value in config["build"].get("exclude", [])]
with tarfile.open(output, "w:gz", dereference=False) as package:
for file in sorted(source.rglob("*")):
relative = file.relative_to(source)
name = PurePosixPath(relative.as_posix())
if any(name == item or item in name.parents for item in excluded):
continue
if any(part in (".git", ".env") or part.startswith(".env.") for part in name.parts):
continue
if file.is_symlink():
raise PipelineError("Symlinks im Build-Verzeichnis sind nicht erlaubt.")
package.add(file, arcname=name.as_posix(), recursive=False)
Path(str(output) + ".sha256").write_text(digest(output) + "\n", encoding="ascii")
inspect_package(output, config, revision)
return revision
def render_remote(config):
remote = config.get("remote", {})
text = (Path(__file__).with_name("remote.sh")).read_text(encoding="utf-8")
hooks = {}
for phase in ("backup", "activate"):
hooks[phase] = "\n".join(
" ".join('"$php"' if arg == "@php" else shlex.quote(arg) for arg in command)
for command in remote.get(phase, [])
) or ":"
mounts = "\n".join(
f"link_shared {shlex.quote(destination)} {shlex.quote(source)}"
for destination, source in remote.get("shared", {}).items()
)
return text.replace("# PUBLIC_PATH", "public_path=" + shlex.quote(remote.get("public", "public"))).replace(
"# BACKUP_HOOKS", hooks["backup"]
).replace("# ACTIVATE_HOOKS", hooks["activate"]).replace("# SHARED_LINKS", mounts or ":")
class Transport:
def __init__(self, config, target, key, known_hosts):
host = os.environ.get("DEPLOY_HOST", "")
user = os.environ.get("DEPLOY_USER", "")
if not NAME.fullmatch(host) or not re.fullmatch(r"[a-zA-Z0-9_-]+", user):
raise PipelineError("DEPLOY_HOST und DEPLOY_USER fehlen oder sind ungültig.")
port = os.environ.get("DEPLOY_PORT", "22")
if not port.isdigit() or not 1 <= int(port) <= 65535:
raise PipelineError("Ungültiger SSH-Port.")
self.options = ["ssh", "-p", port, "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes", "-o",
f"UserKnownHostsFile={known_hosts}", "-i", str(key), f"{user}@{host}"]
self.target = target
self.remote = render_remote(config)
def call(self, command, *arguments, stdout=None):
argv = ["bash", "-s", "--", self.target["path"], self.target.get("php", "/opt/RZphp84/bin/php-cli"), command, *arguments]
return execute([*self.options, shlex.join(argv)], input=self.remote.encode(), stdout=stdout)
def upload(self, archive, release):
destination = f"{self.target['path']}/upload-{release}.tgz"
with open(archive, "rb") as source:
execute([*self.options, "cat > " + shlex.quote(destination)], stdin=source)
def health(url):
for attempt in range(5):
try:
with urllib.request.urlopen(url, timeout=15) as response:
if response.status == 200:
return True
except OSError:
pass
if attempt < 4:
time.sleep(3)
return False
def deploy(project, config, environment, archive, backup_root, transport_factory=Transport):
target = config.get("targets", {}).get(environment)
if not target or not target.get("enabled", False):
raise PipelineError("Ziel ist gesperrt oder nicht konfiguriert.")
expected_branch = target.get("branch")
ref = os.environ.get("GITHUB_REF", os.environ.get("GITEA_REF", ""))
if ref != f"refs/heads/{expected_branch}":
raise PipelineError("Branch und Ziel stimmen nicht überein.")
revision = git(project, "rev-parse", "HEAD")
if revision != os.environ.get("GITHUB_SHA", os.environ.get("GITEA_SHA", "")):
raise PipelineError("Checkout und Workflow-Commit stimmen nicht überein.")
current = git(project, "ls-remote", "origin", ref).split()
if not current or current[0] != revision:
raise PipelineError("Veralteter Auftrag; Branch wurde inzwischen weitergeschoben.")
checksum = inspect_package(archive, config, revision)
key_text = os.environ.get("DEPLOY_SSH_KEY", "")
known_text = os.environ.get("DEPLOY_KNOWN_HOSTS", "")
if not key_text.strip() or not known_text.strip():
raise PipelineError("SSH-Schlüssel und geprüfte Hostschlüssel fehlen.")
if not backup_root or not Path(backup_root).is_dir():
raise PipelineError("Persistenter Sicherungsordner fehlt.")
release = time.strftime("%Y%m%d%H%M%S", time.gmtime()) + "-" + revision[:8]
backup = Path(backup_root) / config["project"] / environment / f"{release}.tar.gz"
backup.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
with tempfile.TemporaryDirectory(prefix="pipeline-ssh-") as temp:
key = Path(temp) / "key"
known_hosts = Path(temp) / "known_hosts"
key.write_text(key_text.rstrip() + "\n", encoding="utf-8")
key.chmod(0o600)
known_hosts.write_text(known_text.rstrip() + "\n", encoding="utf-8")
transport = transport_factory(config, target, key, known_hosts)
transport.call("lock")
try:
with open(backup, "xb") as destination:
backup.chmod(0o600)
transport.call("backup", stdout=destination)
with tarfile.open(backup, "r:gz") as saved:
if not saved.getmembers():
raise PipelineError("Sicherung ist leer.")
transport.call("cleanup")
transport.upload(archive, release)
transport.call("receive", release, checksum)
transport.call("activate", release, revision)
if not health(target["url"].rstrip("/") + target.get("health_path", "/up")):
transport.call("rollback")
raise PipelineError("Gesundheitscheck fehlgeschlagen; Rückfall ausgeführt (Erstinstallation gesperrt).")
transport.call("cleanup")
finally:
transport.call("unlock")
print(f"Ausgeliefert: {config['project']} / {environment} / {revision}")
def main():
parser = argparse.ArgumentParser(description="Gemeinsame Vorbereitung und Strato-Deployments")
parser.add_argument("--version", action="version", version=VERSION)
parser.add_argument("command", choices=["check", "prepare", "deploy"])
parser.add_argument("--project", default=".")
parser.add_argument("--config", default="pipeline.toml")
parser.add_argument("--artifact", default=".pipeline/release.tgz")
parser.add_argument("--environment", choices=["staging", "production"])
parser.add_argument("--backup-root", default=os.environ.get("PIPELINE_BACKUP_ROOT"))
args = parser.parse_args()
try:
config = load_config(Path(args.project) / args.config)
if args.command == "prepare":
prepare(args.project, config, args.artifact)
elif args.command == "deploy":
deploy(args.project, config, args.environment, args.artifact, args.backup_root)
else:
if args.environment and not config.get("targets", {}).get(args.environment, {}).get("enabled", False):
print("Ziel ist gesperrt; kein Build oder Deployment.")
return 3
print(f"Konfiguration gültig: {config['project']} / Manager {VERSION}")
return 0
except (PipelineError, subprocess.CalledProcessError, OSError, ValueError, KeyError, tarfile.TarError) as error:
print(f"Pipeline fehlgeschlagen: {error}")
return 1
if __name__ == "__main__":
raise SystemExit(main())