2929import time
3030import zipfile
3131import io
32+ import glob
3233from datetime import datetime
3334
3435import boto3
@@ -62,13 +63,44 @@ def save_to_file(mapping, filename=None, format="zip"):
6263 json .dump (mapping , file , indent = 2 )
6364
6465
65- def save_to_s3 (data , filename = None , bucket = "smartapi" , format = "zip" ):
66+ def cleanup_backup_files (pattern = "smartapi_????????.zip" , keep_last = 10 , directory = "." ):
67+ """
68+ Clean up old local backup files, keeping only the most recent ones.
69+ :param pattern: Glob pattern (relative to directory) matching the backup files
70+ :param keep_last: Number of most recent files to keep (must be >= 1)
71+ :param directory: Directory containing the backup files
72+ :return: tuple (number of files kept, number of files deleted)
73+ """
74+ if keep_last < 1 :
75+ raise ValueError ("keep_last must be at least 1" )
76+
77+ # File names embed the date (smartapi_YYYYMMDD.zip), so a reverse
78+ # lexicographic sort puts the newest first.
79+ backup_files = sorted (glob .glob (os .path .join (directory , pattern )), reverse = True )
80+
81+ files_to_keep = backup_files [:keep_last ]
82+ files_to_delete = backup_files [keep_last :]
83+
84+ deleted_count = 0
85+ for old_file in files_to_delete :
86+ if os .path .exists (old_file ):
87+ os .remove (old_file )
88+ logging .info (f"Deleted old backup file: { old_file } " )
89+ deleted_count += 1
90+
91+ return len (files_to_keep ), deleted_count
92+
93+
94+ def save_to_s3 (data , filename = None , bucket = "smartapi" , format = "zip" , keep_last = 10 ):
6695 """
6796 Save data to S3 in either JSON or ZIP format.
97+ For the ZIP format a local copy is written first; after a successful upload,
98+ only the `keep_last` most recent local backup zips are kept.
6899 :param data: Data to save
69100 :param filename: File name
70101 :param bucket: S3 bucket name
71102 :param format: File format, either 'json' or 'zip'
103+ :param keep_last: Number of local zip backups to keep after a successful upload
72104 """
73105 filename = filename or _default_filename (f".{ format } " )
74106 s3 = boto3 .resource ("s3" )
@@ -78,7 +110,12 @@ def save_to_s3(data, filename=None, bucket="smartapi", format="zip"):
78110 json_data = json .dumps (data , indent = 2 )
79111 zfile .writestr (filename .replace (".zip" , ".json" ), json_data )
80112 logging .info (f"Uploading { filename } to AWS S3" )
113+ # If the upload raises, we never reach the cleanup below,
114+ # so no local backup is deleted after a failed upload.
81115 s3 .Bucket (bucket ).upload_file (Filename = filename , Key = f"db_backup/{ filename } " )
116+ logging .info (f"Upload of { filename } to S3 successful" )
117+ kept , deleted = cleanup_backup_files (directory = os .path .dirname (filename ) or "." , keep_last = keep_last )
118+ logging .info (f"Backup file cleanup: kept { kept } recent files, deleted { deleted } old files" )
82119 else :
83120 logging .info (f"Uploading { filename } to AWS S3" )
84121 s3 .Bucket (bucket ).put_object (Key = f"db_backup/{ filename } " , Body = json .dumps (data , indent = 2 ))
@@ -105,9 +142,9 @@ def backup_to_file(filename=None, format="zip"):
105142 save_to_file (smartapis , filename , format )
106143
107144
108- def backup_to_s3 (filename = None , bucket = "smartapi" , format = "zip" ):
145+ def backup_to_s3 (filename = None , bucket = "smartapi" , format = "zip" , keep_last = 10 ):
109146 smartapis = _backup ()
110- save_to_s3 (smartapis , filename , bucket , format )
147+ save_to_s3 (smartapis , filename , bucket , format , keep_last = keep_last )
111148
112149
113150def _restore (smartapis ):
0 commit comments