PipelineManager 0.1: gemeinsame Docker-Werkzeuge und Strato-Deployment-Vertrag
PipelineManager Image / image (push) Successful in 2m44s
PipelineManager Image / image (push) Successful in 2m44s
This commit is contained in:
commit
a6e159d125
12 files changed
+807
No files matched your search
@@ -0,0 +1 @@
|
||||
"""Shared Strato deployment tooling; no application or tenant data."""
|
||||
+300
@@ -0,0 +1,300 @@
|
||||
"""Pipeline contract v1. Python standard library only; applications supply TOML and argv hooks."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
import time
|
||||
import tomllib
|
||||
import urllib.request
|
||||
|
||||
VERSION = "0.1.0"
|
||||
NAME = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_.-]*$")
|
||||
COMMIT = re.compile(r"^[0-9a-f]{40}$")
|
||||
|
||||
|
||||
class PipelineError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def relative_path(value):
|
||||
if not isinstance(value, str) or not value or "\\" in value:
|
||||
raise PipelineError("Ungültiger relativer Pfad.")
|
||||
path = PurePosixPath(value)
|
||||
if path.is_absolute() or any(part in ("..", ".") for part in value.split("/")):
|
||||
raise PipelineError("Absolute Pfade und Pfadwechsel sind nicht erlaubt.")
|
||||
return value
|
||||
|
||||
|
||||
def commands(value):
|
||||
if not isinstance(value, list) or any(
|
||||
not isinstance(command, list) or not command
|
||||
or any(not isinstance(arg, str) or not arg or "\x00" in arg for arg in command)
|
||||
for command in value
|
||||
):
|
||||
raise PipelineError("Befehle müssen Listen von Argumentlisten sein.")
|
||||
return value
|
||||
|
||||
|
||||
def load_config(path):
|
||||
with open(path, "rb") as source:
|
||||
config = tomllib.load(source)
|
||||
if config.get("schema") != 1 or not NAME.fullmatch(config.get("project", "")):
|
||||
raise PipelineError("Projektname oder Konfigurationsversion ungültig.")
|
||||
if config.get("manager") != VERSION:
|
||||
raise PipelineError("Dieses Projekt benötigt eine andere Manager-Version.")
|
||||
build = config.get("build", {})
|
||||
for phase in ("prepare", "test", "finalize"):
|
||||
commands(build.get(phase, []))
|
||||
if not build.get("test"):
|
||||
raise PipelineError("Mindestens ein Testbefehl ist erforderlich.")
|
||||
for item in build.get("exclude", []):
|
||||
relative_path(item)
|
||||
for item in build.get("required", []):
|
||||
relative_path(item)
|
||||
for target in config.get("targets", {}).values():
|
||||
if not isinstance(target.get("enabled", False), bool):
|
||||
raise PipelineError("enabled muss true oder false sein.")
|
||||
directory = target.get("path", "")
|
||||
if not directory or any(not NAME.fullmatch(part) or part in (".", "..") for part in directory.split("/")):
|
||||
raise PipelineError("Unsicheres Zielverzeichnis.")
|
||||
if not target.get("url", "").startswith("https://"):
|
||||
raise PipelineError("Gesundheitscheck benötigt HTTPS.")
|
||||
remote = config.get("remote", {})
|
||||
relative_path(remote.get("public", "public"))
|
||||
for destination, source in remote.get("shared", {}).items():
|
||||
relative_path(destination)
|
||||
relative_path(source)
|
||||
commands(remote.get("backup", []))
|
||||
commands(remote.get("activate", []))
|
||||
return config
|
||||
|
||||
|
||||
def execute(argv, cwd=None, env=None, **options):
|
||||
return subprocess.run(argv, cwd=cwd, env=env, check=True, **options)
|
||||
|
||||
|
||||
def git(project, *arguments):
|
||||
return execute(["git", "-c", f"safe.directory={Path(project).resolve()}", "-C", str(project), *arguments], capture_output=True, text=True).stdout.strip()
|
||||
|
||||
|
||||
def digest(path):
|
||||
with open(path, "rb") as source:
|
||||
return hashlib.file_digest(source, "sha256").hexdigest()
|
||||
|
||||
|
||||
def inspect_package(archive, config, revision):
|
||||
checksum = Path(str(archive) + ".sha256")
|
||||
expected = checksum.read_text().strip()
|
||||
if not re.fullmatch(r"[0-9a-f]{64}", expected) or digest(archive) != expected:
|
||||
raise PipelineError("Paket-Prüfsumme stimmt nicht.")
|
||||
with tarfile.open(archive, "r:gz") as package:
|
||||
names = set()
|
||||
for member in package.getmembers():
|
||||
name = relative_path(member.name)
|
||||
if member.issym() or member.islnk() or not (member.isfile() or member.isdir()):
|
||||
raise PipelineError("Links und Spezialdateien sind im Paket nicht erlaubt.")
|
||||
if any(part == ".git" or part == ".env" or part.startswith(".env.") for part in PurePosixPath(name).parts):
|
||||
raise PipelineError("Geheimnisdatei im Paket.")
|
||||
if name in names:
|
||||
raise PipelineError("Doppelter Paketpfad.")
|
||||
names.add(name)
|
||||
metadata = json.load(package.extractfile("PIPELINE.json"))
|
||||
if metadata != {"schema": 1, "project": config["project"], "revision": revision, "tests": "passed"}:
|
||||
raise PipelineError("Paket gehört nicht zum geprüften Projekt und Commit.")
|
||||
if not set(config["build"].get("required", [])).issubset(names):
|
||||
raise PipelineError("Pflichtdateien fehlen im Paket.")
|
||||
return expected
|
||||
|
||||
|
||||
def prepare(project, config, output):
|
||||
project = Path(project).resolve()
|
||||
revision = git(project, "rev-parse", "HEAD")
|
||||
if not COMMIT.fullmatch(revision):
|
||||
raise PipelineError("Ungültige Git-Revision.")
|
||||
output = Path(output).resolve()
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
if output.exists():
|
||||
raise PipelineError("Ausgabepaket existiert bereits.")
|
||||
with tempfile.TemporaryDirectory(prefix="pipeline-build-") as temp:
|
||||
work = Path(temp)
|
||||
export = work / "source.tar"
|
||||
execute(["git", "-C", str(project), "archive", "--output", str(export), revision])
|
||||
source = work / "source"
|
||||
source.mkdir()
|
||||
with tarfile.open(export) as package:
|
||||
for member in package.getmembers():
|
||||
relative_path(member.name.rstrip("/"))
|
||||
if member.issym() or member.islnk() or not (member.isfile() or member.isdir()):
|
||||
raise PipelineError("Git-Export enthält einen Link oder eine Spezialdatei.")
|
||||
package.extractall(source)
|
||||
environment = os.environ.copy()
|
||||
# Deployment secrets are supplied only to the later deployment step.
|
||||
for key in list(environment):
|
||||
if key.startswith(("DEPLOY_", "SSH_")):
|
||||
del environment[key]
|
||||
environment.update({str(key): str(value) for key, value in config["build"].get("environment", {}).items()})
|
||||
for phase in ("prepare", "test", "finalize"):
|
||||
print(f"Phase: {phase}", flush=True)
|
||||
for command in config["build"].get(phase, []):
|
||||
execute(command, cwd=source, env=environment)
|
||||
metadata = {"schema": 1, "project": config["project"], "revision": revision, "tests": "passed"}
|
||||
(source / "PIPELINE.json").write_text(json.dumps(metadata), encoding="utf-8")
|
||||
excluded = [PurePosixPath(value) for value in config["build"].get("exclude", [])]
|
||||
with tarfile.open(output, "w:gz", dereference=False) as package:
|
||||
for file in sorted(source.rglob("*")):
|
||||
relative = file.relative_to(source)
|
||||
name = PurePosixPath(relative.as_posix())
|
||||
if any(name == item or item in name.parents for item in excluded):
|
||||
continue
|
||||
if any(part in (".git", ".env") or part.startswith(".env.") for part in name.parts):
|
||||
continue
|
||||
if file.is_symlink():
|
||||
raise PipelineError("Symlinks im Build-Verzeichnis sind nicht erlaubt.")
|
||||
package.add(file, arcname=name.as_posix(), recursive=False)
|
||||
Path(str(output) + ".sha256").write_text(digest(output) + "\n", encoding="ascii")
|
||||
inspect_package(output, config, revision)
|
||||
return revision
|
||||
|
||||
|
||||
def render_remote(config):
|
||||
remote = config.get("remote", {})
|
||||
text = (Path(__file__).with_name("remote.sh")).read_text(encoding="utf-8")
|
||||
hooks = {}
|
||||
for phase in ("backup", "activate"):
|
||||
hooks[phase] = "\n".join(
|
||||
" ".join('"$php"' if arg == "@php" else shlex.quote(arg) for arg in command)
|
||||
for command in remote.get(phase, [])
|
||||
) or ":"
|
||||
mounts = "\n".join(
|
||||
f"link_shared {shlex.quote(destination)} {shlex.quote(source)}"
|
||||
for destination, source in remote.get("shared", {}).items()
|
||||
)
|
||||
return text.replace("# PUBLIC_PATH", "public_path=" + shlex.quote(remote.get("public", "public"))).replace(
|
||||
"# BACKUP_HOOKS", hooks["backup"]
|
||||
).replace("# ACTIVATE_HOOKS", hooks["activate"]).replace("# SHARED_LINKS", mounts or ":")
|
||||
|
||||
|
||||
class Transport:
|
||||
def __init__(self, config, target, key, known_hosts):
|
||||
host = os.environ.get("DEPLOY_HOST", "")
|
||||
user = os.environ.get("DEPLOY_USER", "")
|
||||
if not NAME.fullmatch(host) or not re.fullmatch(r"[a-zA-Z0-9_-]+", user):
|
||||
raise PipelineError("DEPLOY_HOST und DEPLOY_USER fehlen oder sind ungültig.")
|
||||
port = os.environ.get("DEPLOY_PORT", "22")
|
||||
if not port.isdigit() or not 1 <= int(port) <= 65535:
|
||||
raise PipelineError("Ungültiger SSH-Port.")
|
||||
self.options = ["ssh", "-p", port, "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes", "-o",
|
||||
f"UserKnownHostsFile={known_hosts}", "-i", str(key), f"{user}@{host}"]
|
||||
self.target = target
|
||||
self.remote = render_remote(config)
|
||||
|
||||
def call(self, command, *arguments, stdout=None):
|
||||
argv = ["bash", "-s", "--", self.target["path"], self.target.get("php", "/opt/RZphp84/bin/php-cli"), command, *arguments]
|
||||
return execute([*self.options, shlex.join(argv)], input=self.remote.encode(), stdout=stdout)
|
||||
|
||||
def upload(self, archive, release):
|
||||
destination = f"{self.target['path']}/upload-{release}.tgz"
|
||||
with open(archive, "rb") as source:
|
||||
execute([*self.options, "cat > " + shlex.quote(destination)], stdin=source)
|
||||
|
||||
|
||||
def health(url):
|
||||
for attempt in range(5):
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=15) as response:
|
||||
if response.status == 200:
|
||||
return True
|
||||
except OSError:
|
||||
pass
|
||||
if attempt < 4:
|
||||
time.sleep(3)
|
||||
return False
|
||||
|
||||
|
||||
def deploy(project, config, environment, archive, backup_root, transport_factory=Transport):
|
||||
target = config.get("targets", {}).get(environment)
|
||||
if not target or not target.get("enabled", False):
|
||||
raise PipelineError("Ziel ist gesperrt oder nicht konfiguriert.")
|
||||
expected_branch = target.get("branch")
|
||||
ref = os.environ.get("GITHUB_REF", os.environ.get("GITEA_REF", ""))
|
||||
if ref != f"refs/heads/{expected_branch}":
|
||||
raise PipelineError("Branch und Ziel stimmen nicht überein.")
|
||||
revision = git(project, "rev-parse", "HEAD")
|
||||
if revision != os.environ.get("GITHUB_SHA", os.environ.get("GITEA_SHA", "")):
|
||||
raise PipelineError("Checkout und Workflow-Commit stimmen nicht überein.")
|
||||
current = git(project, "ls-remote", "origin", ref).split()
|
||||
if not current or current[0] != revision:
|
||||
raise PipelineError("Veralteter Auftrag; Branch wurde inzwischen weitergeschoben.")
|
||||
checksum = inspect_package(archive, config, revision)
|
||||
key_text = os.environ.get("DEPLOY_SSH_KEY", "")
|
||||
known_text = os.environ.get("DEPLOY_KNOWN_HOSTS", "")
|
||||
if not key_text.strip() or not known_text.strip():
|
||||
raise PipelineError("SSH-Schlüssel und geprüfte Hostschlüssel fehlen.")
|
||||
if not backup_root or not Path(backup_root).is_dir():
|
||||
raise PipelineError("Persistenter Sicherungsordner fehlt.")
|
||||
release = time.strftime("%Y%m%d%H%M%S", time.gmtime()) + "-" + revision[:8]
|
||||
backup = Path(backup_root) / config["project"] / environment / f"{release}.tar.gz"
|
||||
backup.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
with tempfile.TemporaryDirectory(prefix="pipeline-ssh-") as temp:
|
||||
key = Path(temp) / "key"
|
||||
known_hosts = Path(temp) / "known_hosts"
|
||||
key.write_text(key_text.rstrip() + "\n", encoding="utf-8")
|
||||
key.chmod(0o600)
|
||||
known_hosts.write_text(known_text.rstrip() + "\n", encoding="utf-8")
|
||||
transport = transport_factory(config, target, key, known_hosts)
|
||||
transport.call("lock")
|
||||
try:
|
||||
with open(backup, "xb") as destination:
|
||||
backup.chmod(0o600)
|
||||
transport.call("backup", stdout=destination)
|
||||
with tarfile.open(backup, "r:gz") as saved:
|
||||
if not saved.getmembers():
|
||||
raise PipelineError("Sicherung ist leer.")
|
||||
transport.call("cleanup")
|
||||
transport.upload(archive, release)
|
||||
transport.call("receive", release, checksum)
|
||||
transport.call("activate", release, revision)
|
||||
if not health(target["url"].rstrip("/") + target.get("health_path", "/up")):
|
||||
transport.call("rollback")
|
||||
raise PipelineError("Gesundheitscheck fehlgeschlagen; Rückfall ausgeführt (Erstinstallation gesperrt).")
|
||||
transport.call("cleanup")
|
||||
finally:
|
||||
transport.call("unlock")
|
||||
print(f"Ausgeliefert: {config['project']} / {environment} / {revision}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Gemeinsame Vorbereitung und Strato-Deployments")
|
||||
parser.add_argument("--version", action="version", version=VERSION)
|
||||
parser.add_argument("command", choices=["check", "prepare", "deploy"])
|
||||
parser.add_argument("--project", default=".")
|
||||
parser.add_argument("--config", default="pipeline.toml")
|
||||
parser.add_argument("--artifact", default=".pipeline/release.tgz")
|
||||
parser.add_argument("--environment", choices=["staging", "production"])
|
||||
parser.add_argument("--backup-root", default=os.environ.get("PIPELINE_BACKUP_ROOT"))
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
config = load_config(Path(args.project) / args.config)
|
||||
if args.command == "prepare":
|
||||
prepare(args.project, config, args.artifact)
|
||||
elif args.command == "deploy":
|
||||
deploy(args.project, config, args.environment, args.artifact, args.backup_root)
|
||||
else:
|
||||
if args.environment and not config.get("targets", {}).get(args.environment, {}).get("enabled", False):
|
||||
print("Ziel ist gesperrt; kein Build oder Deployment.")
|
||||
return 3
|
||||
print(f"Konfiguration gültig: {config['project']} / Manager {VERSION}")
|
||||
return 0
|
||||
except (PipelineError, subprocess.CalledProcessError, OSError, ValueError, KeyError, tarfile.TarError) as error:
|
||||
print(f"Pipeline fehlgeschlagen: {error}")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
base="$1"; php="$2"; command="$3"; shift 3
|
||||
# PUBLIC_PATH
|
||||
[[ "$base" =~ ^[a-zA-Z0-9_./-]+$ && "$base" != /* && "$base" != *..* && "$base" != '.' && ! -L "$base" ]] || exit 1
|
||||
mkdir -p "$base"
|
||||
cd "$base"
|
||||
base="$(pwd -P)"
|
||||
|
||||
active_release() {
|
||||
local active
|
||||
active="$(readlink current 2>/dev/null || true)"
|
||||
[[ -z "$active" || "$active" =~ ^releases/[a-zA-Z0-9_-]+$ ]] || exit 1
|
||||
printf '%s' "$active"
|
||||
}
|
||||
|
||||
switch_to() {
|
||||
local target="$1" version web_php
|
||||
[[ "$target" =~ ^releases/[a-zA-Z0-9_-]+$ && -d "$target/$public_path" ]] || exit 1
|
||||
version="${target#releases/}"
|
||||
web_php="$(printf '%s' "$php" | grep -oE 'RZphp[0-9]+' | grep -oE '[0-9]+' || true)"
|
||||
{
|
||||
[[ -z "$web_php" ]] || echo "AddType application/x-httpd-php$web_php .php"
|
||||
cat <<HTACCESS
|
||||
<IfModule mod_rewrite.c>
|
||||
RewriteEngine On
|
||||
RewriteRule ^(releases/(?!$version/$public_path/)|shared(/|$)|current(/|$)|\.|upload-) - [F,L]
|
||||
RewriteCond %{REQUEST_URI} !^/releases/$version/$public_path/
|
||||
RewriteRule ^(.*)\$ releases/$version/$public_path/\$1 [L]
|
||||
</IfModule>
|
||||
<IfModule !mod_rewrite.c>
|
||||
Require all denied
|
||||
</IfModule>
|
||||
HTACCESS
|
||||
} > .htaccess.new
|
||||
ln -sfn "$target" current.new
|
||||
mv -Tf current.new current
|
||||
mv -f .htaccess.new .htaccess
|
||||
}
|
||||
|
||||
link_shared() {
|
||||
local destination="$1" source="$2"
|
||||
[[ -e "$base/shared/$source" ]] || { echo "Persistenter Pfad fehlt: shared/$source" >&2; exit 1; }
|
||||
mkdir -p "$(dirname "$dir/$destination")"
|
||||
rm -rf "$dir/$destination"
|
||||
ln -s "$base/shared/$source" "$dir/$destination"
|
||||
}
|
||||
|
||||
case "$command" in
|
||||
lock)
|
||||
mkdir -p releases
|
||||
mkdir .deploy-lock || { echo 'Deployment läuft bereits; .deploy-lock prüfen.' >&2; exit 1; }
|
||||
;;
|
||||
unlock) rmdir .deploy-lock ;;
|
||||
backup)
|
||||
if [[ -n "$(active_release)" ]]; then
|
||||
(
|
||||
cd current
|
||||
# BACKUP_HOOKS
|
||||
) >&2
|
||||
fi
|
||||
tar -czf - shared
|
||||
;;
|
||||
receive)
|
||||
release="$1"; checksum="$2"
|
||||
[[ "$release" =~ ^[a-zA-Z0-9_-]+$ && "$checksum" =~ ^[0-9a-f]{64}$ ]] || exit 1
|
||||
[[ "$(sha256sum "upload-$release.tgz" | cut -d ' ' -f1)" == "$checksum" ]] || exit 1
|
||||
mkdir "releases/$release"
|
||||
tar --no-same-owner -xzf "upload-$release.tgz" -C "releases/$release"
|
||||
rm "upload-$release.tgz"
|
||||
;;
|
||||
activate)
|
||||
release="$1"; revision="$2"
|
||||
[[ "$release" =~ ^[a-zA-Z0-9_-]+$ && "$revision" =~ ^[0-9a-f]{40}$ ]] || exit 1
|
||||
dir="releases/$release"
|
||||
[[ -d "$dir/$public_path" ]] || exit 1
|
||||
# SHARED_LINKS
|
||||
printf '%s\n' "$revision" > "$dir/REVISION"
|
||||
(
|
||||
cd "$dir"
|
||||
# ACTIVATE_HOOKS
|
||||
)
|
||||
active_release > .previous_release
|
||||
switch_to "$dir"
|
||||
;;
|
||||
rollback)
|
||||
target="$(cat .previous_release 2>/dev/null || true)"
|
||||
if [[ -z "$target" ]]; then
|
||||
echo 'Erstinstallation fehlgeschlagen; Zugriff gesperrt.' >&2
|
||||
printf 'Require all denied\n' > .htaccess
|
||||
[[ ! -L current ]] || rm current
|
||||
exit 0
|
||||
fi
|
||||
switch_to "$target"
|
||||
;;
|
||||
cleanup)
|
||||
active="$(active_release)"
|
||||
for dir in releases/*; do
|
||||
if [[ -d "$dir" && "$dir" != "$active" ]]; then
|
||||
[[ "$dir" =~ ^releases/[a-zA-Z0-9_-]+$ && ! -L "$dir" ]] || exit 1
|
||||
rm -rf "$dir"
|
||||
fi
|
||||
done
|
||||
for archive in upload-*.tgz; do
|
||||
[[ ! -f "$archive" ]] || rm -f "$archive"
|
||||
done
|
||||
;;
|
||||
*) echo 'Unbekannter Remote-Befehl.' >&2; exit 1 ;;
|
||||
esac
|
||||
Reference in new issue
Block a user