Skip to content

Commit 155806b

Browse files
fix: auto delete backup files.
1 parent 2322de7 commit 155806b

1 file changed

Lines changed: 40 additions & 3 deletions

File tree

‎src/admin.py‎

Lines changed: 40 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import time
3030
import zipfile
3131
import io
32+
import glob
3233
from datetime import datetime
3334

3435
import boto3
@@ -62,13 +63,44 @@ def save_to_file(mapping, filename=None, format="zip"):
6263
json.dump(mapping, file, indent=2)
6364

6465

65-
def save_to_s3(data, filename=None, bucket="smartapi", format="zip"):
66+
def cleanup_backup_files(pattern="smartapi_????????.zip", keep_last=10, directory="."):
67+
"""
68+
Clean up old local backup files, keeping only the most recent ones.
69+
:param pattern: Glob pattern (relative to directory) matching the backup files
70+
:param keep_last: Number of most recent files to keep (must be >= 1)
71+
:param directory: Directory containing the backup files
72+
:return: tuple (number of files kept, number of files deleted)
73+
"""
74+
if keep_last < 1:
75+
raise ValueError("keep_last must be at least 1")
76+
77+
# File names embed the date (smartapi_YYYYMMDD.zip), so a reverse
78+
# lexicographic sort puts the newest first.
79+
backup_files = sorted(glob.glob(os.path.join(directory, pattern)), reverse=True)
80+
81+
files_to_keep = backup_files[:keep_last]
82+
files_to_delete = backup_files[keep_last:]
83+
84+
deleted_count = 0
85+
for old_file in files_to_delete:
86+
if os.path.exists(old_file):
87+
os.remove(old_file)
88+
logging.info(f"Deleted old backup file: {old_file}")
89+
deleted_count += 1
90+
91+
return len(files_to_keep), deleted_count
92+
93+
94+
def save_to_s3(data, filename=None, bucket="smartapi", format="zip", keep_last=10):
6695
"""
6796
Save data to S3 in either JSON or ZIP format.
97+
For the ZIP format a local copy is written first; after a successful upload,
98+
only the `keep_last` most recent local backup zips are kept.
6899
:param data: Data to save
69100
:param filename: File name
70101
:param bucket: S3 bucket name
71102
:param format: File format, either 'json' or 'zip'
103+
:param keep_last: Number of local zip backups to keep after a successful upload
72104
"""
73105
filename = filename or _default_filename(f".{format}")
74106
s3 = boto3.resource("s3")
@@ -78,7 +110,12 @@ def save_to_s3(data, filename=None, bucket="smartapi", format="zip"):
78110
json_data = json.dumps(data, indent=2)
79111
zfile.writestr(filename.replace(".zip", ".json"), json_data)
80112
logging.info(f"Uploading {filename} to AWS S3")
113+
# If the upload raises, we never reach the cleanup below,
114+
# so no local backup is deleted after a failed upload.
81115
s3.Bucket(bucket).upload_file(Filename=filename, Key=f"db_backup/{filename}")
116+
logging.info(f"Upload of {filename} to S3 successful")
117+
kept, deleted = cleanup_backup_files(directory=os.path.dirname(filename) or ".", keep_last=keep_last)
118+
logging.info(f"Backup file cleanup: kept {kept} recent files, deleted {deleted} old files")
82119
else:
83120
logging.info(f"Uploading {filename} to AWS S3")
84121
s3.Bucket(bucket).put_object(Key=f"db_backup/{filename}", Body=json.dumps(data, indent=2))
@@ -105,9 +142,9 @@ def backup_to_file(filename=None, format="zip"):
105142
save_to_file(smartapis, filename, format)
106143

107144

108-
def backup_to_s3(filename=None, bucket="smartapi", format="zip"):
145+
def backup_to_s3(filename=None, bucket="smartapi", format="zip", keep_last=10):
109146
smartapis = _backup()
110-
save_to_s3(smartapis, filename, bucket, format)
147+
save_to_s3(smartapis, filename, bucket, format, keep_last=keep_last)
111148

112149

113150
def _restore(smartapis):

0 commit comments

Comments
 (0)