310 lines
14 KiB
Python
310 lines
14 KiB
Python
"""Pipeline contract v1. Python standard library only; applications supply TOML and argv hooks."""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path, PurePosixPath
|
|
import re
|
|
import shlex
|
|
import subprocess
|
|
import tarfile
|
|
import tempfile
|
|
import time
|
|
import tomllib
|
|
import urllib.request
|
|
|
|
VERSION = "0.1.1"
|
|
NAME = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_.-]*$")
|
|
COMMIT = re.compile(r"^[0-9a-f]{40}$")
|
|
|
|
|
|
class PipelineError(Exception):
|
|
pass
|
|
|
|
|
|
def relative_path(value):
|
|
if not isinstance(value, str) or not value or "\\" in value:
|
|
raise PipelineError("Ungültiger relativer Pfad.")
|
|
path = PurePosixPath(value)
|
|
if path.is_absolute() or any(part in ("..", ".") for part in value.split("/")):
|
|
raise PipelineError("Absolute Pfade und Pfadwechsel sind nicht erlaubt.")
|
|
return value
|
|
|
|
|
|
def commands(value):
|
|
if not isinstance(value, list) or any(
|
|
not isinstance(command, list) or not command
|
|
or any(not isinstance(arg, str) or not arg or "\x00" in arg for arg in command)
|
|
for command in value
|
|
):
|
|
raise PipelineError("Befehle müssen Listen von Argumentlisten sein.")
|
|
return value
|
|
|
|
|
|
def load_config(path):
|
|
with open(path, "rb") as source:
|
|
config = tomllib.load(source)
|
|
if config.get("schema") != 1 or not NAME.fullmatch(config.get("project", "")):
|
|
raise PipelineError("Projektname oder Konfigurationsversion ungültig.")
|
|
if config.get("manager") != VERSION:
|
|
raise PipelineError("Dieses Projekt benötigt eine andere Manager-Version.")
|
|
build = config.get("build", {})
|
|
for phase in ("prepare", "test", "finalize"):
|
|
commands(build.get(phase, []))
|
|
if not build.get("test"):
|
|
raise PipelineError("Mindestens ein Testbefehl ist erforderlich.")
|
|
for item in build.get("exclude", []):
|
|
relative_path(item)
|
|
for item in build.get("required", []):
|
|
relative_path(item)
|
|
for target in config.get("targets", {}).values():
|
|
if not isinstance(target.get("enabled", False), bool):
|
|
raise PipelineError("enabled muss true oder false sein.")
|
|
directory = target.get("path", "")
|
|
if not directory or any(not NAME.fullmatch(part) or part in (".", "..") for part in directory.split("/")):
|
|
raise PipelineError("Unsicheres Zielverzeichnis.")
|
|
if not target.get("url", "").startswith("https://"):
|
|
raise PipelineError("Gesundheitscheck benötigt HTTPS.")
|
|
remote = config.get("remote", {})
|
|
relative_path(remote.get("public", "public"))
|
|
for destination, source in remote.get("shared", {}).items():
|
|
relative_path(destination)
|
|
relative_path(source)
|
|
commands(remote.get("backup", []))
|
|
commands(remote.get("activate", []))
|
|
return config
|
|
|
|
|
|
def execute(argv, cwd=None, env=None, **options):
|
|
return subprocess.run(argv, cwd=cwd, env=env, check=True, **options)
|
|
|
|
|
|
def git(project, *arguments):
|
|
return execute(["git", "-c", f"safe.directory={Path(project).resolve()}", "-C", str(project), *arguments], capture_output=True, text=True).stdout.strip()
|
|
|
|
|
|
def digest(path):
|
|
with open(path, "rb") as source:
|
|
return hashlib.file_digest(source, "sha256").hexdigest()
|
|
|
|
|
|
def inspect_package(archive, config, revision):
|
|
checksum = Path(str(archive) + ".sha256")
|
|
expected = checksum.read_text().strip()
|
|
if not re.fullmatch(r"[0-9a-f]{64}", expected) or digest(archive) != expected:
|
|
raise PipelineError("Paket-Prüfsumme stimmt nicht.")
|
|
with tarfile.open(archive, "r:gz") as package:
|
|
names = set()
|
|
for member in package.getmembers():
|
|
name = relative_path(member.name)
|
|
if member.issym() or member.islnk() or not (member.isfile() or member.isdir()):
|
|
raise PipelineError("Links und Spezialdateien sind im Paket nicht erlaubt.")
|
|
if any(part == ".git" or part == ".env" or part.startswith(".env.") for part in PurePosixPath(name).parts):
|
|
raise PipelineError("Geheimnisdatei im Paket.")
|
|
if name in names:
|
|
raise PipelineError("Doppelter Paketpfad.")
|
|
names.add(name)
|
|
metadata = json.load(package.extractfile("PIPELINE.json"))
|
|
if metadata != {"schema": 1, "project": config["project"], "revision": revision, "tests": "passed"}:
|
|
raise PipelineError("Paket gehört nicht zum geprüften Projekt und Commit.")
|
|
if not set(config["build"].get("required", [])).issubset(names):
|
|
raise PipelineError("Pflichtdateien fehlen im Paket.")
|
|
return expected
|
|
|
|
|
|
def prepare(project, config, output):
|
|
project = Path(project).resolve()
|
|
revision = git(project, "rev-parse", "HEAD")
|
|
if not COMMIT.fullmatch(revision):
|
|
raise PipelineError("Ungültige Git-Revision.")
|
|
output = Path(output).resolve()
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
if output.exists():
|
|
raise PipelineError("Ausgabepaket existiert bereits.")
|
|
with tempfile.TemporaryDirectory(prefix="pipeline-build-") as temp:
|
|
work = Path(temp)
|
|
export = work / "source.tar"
|
|
execute(["git", "-C", str(project), "archive", "--output", str(export), revision])
|
|
source = work / "source"
|
|
source.mkdir()
|
|
with tarfile.open(export) as package:
|
|
for member in package.getmembers():
|
|
relative_path(member.name.rstrip("/"))
|
|
if member.issym() or member.islnk() or not (member.isfile() or member.isdir()):
|
|
raise PipelineError("Git-Export enthält einen Link oder eine Spezialdatei.")
|
|
package.extractall(source)
|
|
environment = os.environ.copy()
|
|
# Deployment secrets are supplied only to the later deployment step.
|
|
for key in list(environment):
|
|
if key.startswith(("DEPLOY_", "SSH_")):
|
|
del environment[key]
|
|
environment.update({str(key): str(value) for key, value in config["build"].get("environment", {}).items()})
|
|
for phase in ("prepare", "test", "finalize"):
|
|
print(f"Phase: {phase}", flush=True)
|
|
for command in config["build"].get(phase, []):
|
|
execute(command, cwd=source, env=environment)
|
|
metadata = {"schema": 1, "project": config["project"], "revision": revision, "tests": "passed"}
|
|
(source / "PIPELINE.json").write_text(json.dumps(metadata), encoding="utf-8")
|
|
excluded = [PurePosixPath(value) for value in config["build"].get("exclude", [])]
|
|
with tarfile.open(output, "w:gz", dereference=False) as package:
|
|
for file in sorted(source.rglob("*")):
|
|
relative = file.relative_to(source)
|
|
name = PurePosixPath(relative.as_posix())
|
|
if any(name == item or item in name.parents for item in excluded):
|
|
continue
|
|
if any(part in (".git", ".env") or part.startswith(".env.") for part in name.parts):
|
|
continue
|
|
if file.is_symlink():
|
|
raise PipelineError("Symlinks im Build-Verzeichnis sind nicht erlaubt.")
|
|
package.add(file, arcname=name.as_posix(), recursive=False)
|
|
Path(str(output) + ".sha256").write_text(digest(output) + "\n", encoding="ascii")
|
|
inspect_package(output, config, revision)
|
|
return revision
|
|
|
|
|
|
def render_remote(config):
|
|
remote = config.get("remote", {})
|
|
text = (Path(__file__).with_name("remote.sh")).read_text(encoding="utf-8")
|
|
hooks = {}
|
|
for phase in ("backup", "activate"):
|
|
hooks[phase] = "\n".join(
|
|
" ".join('"$php"' if arg == "@php" else shlex.quote(arg) for arg in command)
|
|
for command in remote.get(phase, [])
|
|
) or ":"
|
|
mounts = "\n".join(
|
|
f"link_shared {shlex.quote(destination)} {shlex.quote(source)}"
|
|
for destination, source in remote.get("shared", {}).items()
|
|
)
|
|
return text.replace("# PUBLIC_PATH", "public_path=" + shlex.quote(remote.get("public", "public"))).replace(
|
|
"# BACKUP_HOOKS", hooks["backup"]
|
|
).replace("# ACTIVATE_HOOKS", hooks["activate"]).replace("# SHARED_LINKS", mounts or ":")
|
|
|
|
|
|
class Transport:
|
|
def __init__(self, config, target, key, known_hosts):
|
|
host = os.environ.get("DEPLOY_HOST", "")
|
|
user = os.environ.get("DEPLOY_USER", "")
|
|
if not NAME.fullmatch(host) or not re.fullmatch(r"[a-zA-Z0-9_-]+", user):
|
|
raise PipelineError("DEPLOY_HOST und DEPLOY_USER fehlen oder sind ungültig.")
|
|
port = os.environ.get("DEPLOY_PORT", "22")
|
|
if not port.isdigit() or not 1 <= int(port) <= 65535:
|
|
raise PipelineError("Ungültiger SSH-Port.")
|
|
self.options = ["ssh", "-p", port, "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes", "-o",
|
|
f"UserKnownHostsFile={known_hosts}", "-i", str(key), f"{user}@{host}"]
|
|
self.target = target
|
|
self.remote = render_remote(config)
|
|
|
|
def call(self, command, *arguments, stdout=None):
|
|
argv = ["bash", "-s", "--", self.target["path"], self.target.get("php", "/opt/RZphp84/bin/php-cli"), command, *arguments]
|
|
return execute([*self.options, shlex.join(argv)], input=self.remote.encode(), stdout=stdout)
|
|
|
|
def upload(self, archive, release):
|
|
destination = f"{self.target['path']}/upload-{release}.tgz"
|
|
with open(archive, "rb") as source:
|
|
execute([*self.options, "cat > " + shlex.quote(destination)], stdin=source)
|
|
|
|
|
|
def health(url):
|
|
for attempt in range(5):
|
|
try:
|
|
with urllib.request.urlopen(url, timeout=15) as response:
|
|
if response.status == 200:
|
|
return True
|
|
except OSError:
|
|
pass
|
|
if attempt < 4:
|
|
time.sleep(3)
|
|
return False
|
|
|
|
|
|
def deploy(project, config, environment, archive, backup_root, transport_factory=Transport):
|
|
target = config.get("targets", {}).get(environment)
|
|
if not target or not target.get("enabled", False):
|
|
raise PipelineError("Ziel ist gesperrt oder nicht konfiguriert.")
|
|
expected_branch = target.get("branch")
|
|
ref = os.environ.get("GITHUB_REF", os.environ.get("GITEA_REF", ""))
|
|
if ref != f"refs/heads/{expected_branch}":
|
|
raise PipelineError("Branch und Ziel stimmen nicht überein.")
|
|
revision = git(project, "rev-parse", "HEAD")
|
|
if revision != os.environ.get("GITHUB_SHA", os.environ.get("GITEA_SHA", "")):
|
|
raise PipelineError("Checkout und Workflow-Commit stimmen nicht überein.")
|
|
current = git(project, "ls-remote", "origin", ref).split()
|
|
if not current or current[0] != revision:
|
|
raise PipelineError("Veralteter Auftrag; Branch wurde inzwischen weitergeschoben.")
|
|
if target.get("require_merge", False):
|
|
parents = git(project, "show", "-s", "--format=%P", "HEAD").split()
|
|
if len(parents) != 2:
|
|
raise PipelineError("Dieses Ziel benötigt einen Merge-Commit.")
|
|
if target.get("source_branch"):
|
|
try:
|
|
git(project, "merge-base", "--is-ancestor", parents[1], "origin/" + target["source_branch"])
|
|
except subprocess.CalledProcessError as error:
|
|
raise PipelineError("Merge stammt nicht vom vorgesehenen Entwicklungsbranch.") from error
|
|
checksum = inspect_package(archive, config, revision)
|
|
key_text = os.environ.get("DEPLOY_SSH_KEY", "")
|
|
known_text = os.environ.get("DEPLOY_KNOWN_HOSTS", "")
|
|
if not key_text.strip() or not known_text.strip():
|
|
raise PipelineError("SSH-Schlüssel und geprüfte Hostschlüssel fehlen.")
|
|
if not backup_root or not Path(backup_root).is_dir():
|
|
raise PipelineError("Persistenter Sicherungsordner fehlt.")
|
|
release = time.strftime("%Y%m%d%H%M%S", time.gmtime()) + "-" + revision[:8]
|
|
backup = Path(backup_root) / config["project"] / environment / f"{release}.tar.gz"
|
|
backup.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
with tempfile.TemporaryDirectory(prefix="pipeline-ssh-") as temp:
|
|
key = Path(temp) / "key"
|
|
known_hosts = Path(temp) / "known_hosts"
|
|
key.write_text(key_text.rstrip() + "\n", encoding="utf-8")
|
|
key.chmod(0o600)
|
|
known_hosts.write_text(known_text.rstrip() + "\n", encoding="utf-8")
|
|
transport = transport_factory(config, target, key, known_hosts)
|
|
transport.call("lock")
|
|
try:
|
|
with open(backup, "xb") as destination:
|
|
backup.chmod(0o600)
|
|
transport.call("backup", stdout=destination)
|
|
with tarfile.open(backup, "r:gz") as saved:
|
|
if not saved.getmembers():
|
|
raise PipelineError("Sicherung ist leer.")
|
|
transport.call("cleanup")
|
|
transport.upload(archive, release)
|
|
transport.call("receive", release, checksum)
|
|
transport.call("activate", release, revision)
|
|
if not health(target["url"].rstrip("/") + target.get("health_path", "/up")):
|
|
transport.call("rollback")
|
|
raise PipelineError("Gesundheitscheck fehlgeschlagen; Rückfall ausgeführt (Erstinstallation gesperrt).")
|
|
transport.call("cleanup")
|
|
finally:
|
|
transport.call("unlock")
|
|
print(f"Ausgeliefert: {config['project']} / {environment} / {revision}")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Gemeinsame Vorbereitung und Strato-Deployments")
|
|
parser.add_argument("--version", action="version", version=VERSION)
|
|
parser.add_argument("command", choices=["check", "prepare", "deploy"])
|
|
parser.add_argument("--project", default=".")
|
|
parser.add_argument("--config", default="pipeline.toml")
|
|
parser.add_argument("--artifact", default=".pipeline/release.tgz")
|
|
parser.add_argument("--environment", choices=["staging", "production"])
|
|
parser.add_argument("--backup-root", default=os.environ.get("PIPELINE_BACKUP_ROOT"))
|
|
args = parser.parse_args()
|
|
try:
|
|
config = load_config(Path(args.project) / args.config)
|
|
if args.command == "prepare":
|
|
prepare(args.project, config, args.artifact)
|
|
elif args.command == "deploy":
|
|
deploy(args.project, config, args.environment, args.artifact, args.backup_root)
|
|
else:
|
|
if args.environment and not config.get("targets", {}).get(args.environment, {}).get("enabled", False):
|
|
print("Ziel ist gesperrt; kein Build oder Deployment.")
|
|
return 3
|
|
print(f"Konfiguration gültig: {config['project']} / Manager {VERSION}")
|
|
return 0
|
|
except (PipelineError, subprocess.CalledProcessError, OSError, ValueError, KeyError, tarfile.TarError) as error:
|
|
print(f"Pipeline fehlgeschlagen: {error}")
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|