From f056eebbd49c966fa4a1e6ba3b6a0d3c6cf6d519 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rodolphe=20Br=C3=A9ard?= Date: Thu, 4 Apr 2024 14:07:18 +0200 Subject: [PATCH] First commit --- README.md | 38 ++++++++ pg_maintenance.py | 217 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 255 insertions(+) create mode 100644 README.md create mode 100755 pg_maintenance.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..7035fa2 --- /dev/null +++ b/README.md @@ -0,0 +1,38 @@ +# pg maintenance + +Minimalist periodic maintenance script for PostgreSQL. + + +## Disclaimer and licensing + +This software has been created for personal purposes and may therefore not suit your needs. +However, you may use it under the terms of the [MIT License][mit_license]. + + +## Requirements + +- Python 3 +- psql / pg_dump +- rsync + + +## Actions + +This runs [VACUUM][doc_vacuum] and [ANALYZE][doc_analyze] on a specified database. +If at least one recipient is specified, it also builds a reports and email it to said recipients. + +To know more about routine vacuuming, please read [PostgreSQL's documentation][doc_why_vacuum]. + + +## Basic usage + +``` +pg_maintenance.py --help +pg_maintenance.py "db_name" --destination "user@remote:/your/backup/path" --email-recipient "admin@exemple.org" +``` + + +[mit_license]: https://opensource.org/license/mit +[doc_why_vacuum]: https://www.postgresql.org/docs/current/routine-vacuuming.html +[doc_vacuum]: https://www.postgresql.org/docs/current/sql-vacuum.html +[doc_analyze]: https://www.postgresql.org/docs/current/sql-analyze.html diff --git a/pg_maintenance.py b/pg_maintenance.py new file mode 100755 index 0000000..d7e1daa --- /dev/null +++ b/pg_maintenance.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +# +# Copyright 2024 Rodolphe Bréard +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +import argparse +import os +import smtplib +import subprocess +import tempfile +import time +from datetime import datetime +from email.mime.application import MIMEApplication +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.utils import COMMASPACE, formatdate + +DEFAULT_EMAIL_HOST = "localhost" +DEFAULT_EMAIL_PORT = 25 +DEFAULT_EMAIL_REPORT_SENDER = "PostgreSQL maintenance script " +DEFAULT_PG_ADMIN_USER = "postgres" + + +def init_report(args): + if not args.email_recipient: + return None + report = MIMEMultipart("alternative") + report["Subject"] = f"Database maintenance for `{args.db_name}`" + report["From"] = args.email_sender + report["To"] = COMMASPACE.join(args.email_recipient) + report["Date"] = formatdate(localtime=True) + report["User-Agent"] = "PostgreSQL maintenance script" + report["Content-Language"] = "en-US" + return report + + +def send_report(args, vacuum, analyze, backup_create, backup_upload): + report = init_report(args) + if report is None: + return + vacuum_report = get_cmd_report("Vacuum", vacuum) + analyze_report = get_cmd_report("Analyze", analyze) + backup_report = get_backup_report(backup_create, backup_upload) + report_txt = f"""Database maintenance report for `{args.db_name}`. + +{vacuum_report} +{analyze_report} +{backup_report} +""" + report.attach(MIMEText(report_txt)) + if vacuum.stdout: + attach_file(report, "vacuum.log", vacuum.stdout) + if vacuum.stderr: + attach_file(report, "vacuum.err.log", vacuum.stderr) + if analyze.stdout: + attach_file(report, "analyze.log", analyze.stdout) + if analyze.stderr: + attach_file(report, "analyze.err.log", analyze.stderr) + with smtplib.SMTP(host=args.email_host, port=args.email_port) as smtp: + smtp.send_message(report) + + +def attach_file(report, file_name, file_content): + part = MIMEApplication(file_content) + part["Content-Disposition"] = f'attachment; filename="{file_name}"' + report.attach(part) + + +def run_sql_cmd(args, cmd): + cmd = [ + "psql", + "--quiet", + "--echo-queries", + "--username", + args.db_admin_user, + "--dbname", + args.db_name, + "--command", + cmd, + ] + start = time.time() + ret = subprocess.run(cmd, capture_output=True) + end = time.time() + ret.elapsed_time = round(end - start, 3) + return ret + + +def create_backup(args): + if not args.destination: + return None + output_file = tempfile.NamedTemporaryFile() + cmd = [ + "pg_dump", + "--create", + "--clean", + "--no-unlogged-table-data", + "--compress", + "zstd", + "--file", + output_file.name, + "--username", + args.db_admin_user, + "--dbname", + args.db_name, + ] + start = time.time() + ret = subprocess.run(cmd, capture_output=True) + end = time.time() + ret.elapsed_time = round(end - start, 3) + ret.file_size = os.path.getsize(output_file.name) + ret.file = output_file + return ret + + +def upload_backup(args, backup_create): + if not args.destination: + return None + timestamp = datetime.today().strftime("%Y-%m-%d") + file_name = f"{timestamp}_{args.db_name}.sql.zst" + cmd = ["rsync", backup_create.file.name, os.path.join(args.destination, file_name)] + start = time.time() + ret = subprocess.run(cmd, capture_output=True) + end = time.time() + ret.elapsed_time = round(end - start, 3) + return ret + + +def get_cmd_report(name, cmd_ret): + if cmd_ret.returncode == 0: + return f"{name} completed in {cmd_ret.elapsed_time} s." + else: + return f"""{name} failed: +- error code: {cmd_ret.returncode} +- elapsed time: {cmd_ret.elapsed_time} s""" + + +def get_backup_report(backup_create, backup_upload): + if not args.destination: + return "No backup upload destination found." + if backup_create.returncode != 0: + return f"""Backup creation failed: +- error code: {backup_create.returncode} +- elapsed time: {backup_create.elapsed_time}s""" + ret = f"""Backup creation completed: +- file size: {backup_create.file_size} B +- elapsed time: {backup_create.elapsed_time} s +""" + if backup_upload.returncode != 0: + ret += """Backup upload failed: +- error code: {backup_upload.returncode} +- message: {backup_upload.stderr}""" + else: + ret += f"Backup uploaded in {backup_upload.elapsed_time} s." + return ret + + +def daily_db_maintenance(args): + vacuum = run_sql_cmd(args, "VACUUM VERBOSE") + analyze = run_sql_cmd(args, "ANALYZE VERBOSE") + backup_create = create_backup(args) + backup_upload = upload_backup(args, backup_create) + send_report(args, vacuum, analyze, backup_create, backup_upload) + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument("db_name", help="name of the database to run maintenance on") + parser.add_argument( + "-u", + "--db-admin-user", + default=DEFAULT_PG_ADMIN_USER, + help="name of the PostgreSQL superuser", + ) + parser.add_argument("-d", "--destination", help="backup upload destination") + parser.add_argument( + "--email-host", default=DEFAULT_EMAIL_HOST, help="hostname of the email server" + ) + parser.add_argument( + "--email-port", + type=int, + default=DEFAULT_EMAIL_PORT, + help="email server submission port", + ) + parser.add_argument( + "--email-sender", + default=DEFAULT_EMAIL_REPORT_SENDER, + help="sender of the email report", + ) + parser.add_argument( + "--email-recipient", + nargs="*", + help="recipient of the email report (may be specified multi times)", + ) + return parser.parse_args() + + +if __name__ == "__main__": + args = get_args() + daily_db_maintenance(args)