Update json_merger.py

json_merger.py

@@ -1,27 +1,41 @@
 import os
 import json
 
-def merge_and_update_json(file1, file2, merged_file):
- # Load data from the first file
- with open(file1, 'r') as f1:
- data1 = json.load(f1)
+def load_json(file_path):
+ """ Load JSON data from a file. """
+ with open(file_path, 'r') as f:
+ return json.load(f)
 
- # Load data from the second file
- with open(file2, 'r') as f2:
- data2 = json.load(f2)
+def save_json(file_path, data):
+ """ Save JSON data to a file. """
+ with open(file_path, 'w') as f:
+ json.dump(data, f, indent=4)
 
- # Merge and update data
- merged_data = data2 + [item for item in data1 if item not in data2]
+def clean_data(data):
+ """ Remove 'Last Fetch Time' tag from data. """
+ for item in data:
+ if 'Last Fetch Time' in item:
+ del item['Last Fetch Time']
+ return data
 
- # Write merged data to new JSON file
- with open(merged_file, 'w') as f:
- json.dump(merged_data, f, indent=4)
+def merge_and_update_json(file1, file2, merged_file):
+ # Load and clean data from the first file
+ data1 = clean_data(load_json(file1))
+
+ # Load and clean data from the second file
+ data2 = clean_data(load_json(file2))
+
+ # Merge data, preserving unique items
+ merged_data = data2 + [item for item in data1 if item not in data2]
+
+ # Write merged data to the new JSON file
+ save_json(merged_file, merged_data)
 
-# File paths
+# Get list of existing JSON files in the data folder
 data_folder = 'data'
-file1 = os.path.join(data_folder, 'data1.json')
-file2 = os.path.join(data_folder, 'data2.json')
-merged_file = os.path.join(data_folder, 'merged_data.json')
+files = sorted([f for f in os.listdir(data_folder) if f.endswith('.json')])
+file1, file2 = os.path.join(data_folder, files[-2]), os.path.join(data_folder, files[-1])
+merged_file = os.path.join(data_folder, f'data{len(files)}.json')
 
 # Merge and update JSON files
 merge_and_update_json(file1, file2, merged_file)