diff --git a/.circleci/ansible/cleanup_tlis.yml b/.circleci/ansible/clean_timelines.yml similarity index 100% rename from .circleci/ansible/cleanup_tlis.yml rename to .circleci/ansible/clean_timelines.yml diff --git a/.circleci/ansible/clean_timelines_json.py b/.circleci/ansible/clean_timelines_json.py new file mode 100755 index 0000000000..b8ea4bfb49 --- /dev/null +++ b/.circleci/ansible/clean_timelines_json.py @@ -0,0 +1,46 @@ +import os +import shutil +import sys +import time +import json +from pathlib import Path + +# Dry run is the default, i.e. not do anything +act = len(sys.argv) == 2 and sys.argv[1] == '--act' +print('act=', act) + +move_to_deleted = True + +# result of GET 'https://console.neon.tech/api/v1/admin/projects?order=asc&sort=id&show_deleted=true' +with open('projects.json') as f: + projects = json.load(f) + +projects = projects['data'] +deleted_projects = [p for p in projects if p["deleted"]] +active_projects = [p for p in projects if not p["deleted"]] +print(f"deleted {len(deleted_projects)}, active {len(active_projects)}, total {len(projects)}") +# print(deleted_projects) + +for project_to_delete in deleted_projects: + tenant_id = project_to_delete['tenant'] + tenant_dir = Path(f"/storage/safekeeper/data/{tenant_id}") + timeline_dir = tenant_dir / project_to_delete['timeline_id'] + + if not os.path.exists(timeline_dir): + continue + + if move_to_deleted: + # move to deleted/ + tenant_dir_deleted = (Path(f"/storage/safekeeper/data/deleted/") / tenant_id) + tenant_dir_deleted.mkdir(parents=True, exist_ok=True) + print(f"moving {timeline_dir} to {tenant_dir_deleted}") + if act: + shutil.move(timeline_dir, tenant_dir_deleted) + else: + print(f"removing {timeline_dir}") + if act: + shutil.rmtree(timeline_dir) + + if len(os.listdir(tenant_dir)) == 0: + print(f"removing empty tenant directory {tenant_dir}") + shutil.rmtree(tenant_dir) \ No newline at end of file diff --git a/.circleci/ansible/clean_timelines_json.yml b/.circleci/ansible/clean_timelines_json.yml new file mode 100644 index 0000000000..1c54d14a1a --- /dev/null +++ b/.circleci/ansible/clean_timelines_json.yml @@ -0,0 +1,36 @@ +- name: remove timelines listed in projects.json + hosts: safekeepers + gather_facts: False + remote_user: admin + tasks: + # - name: install acl + # ansible.builtin.apt: + # name: + # - python3-pip + # - acl + # become: True + + - name: upload projects.json + ansible.builtin.copy: + src: /home/ars/zenith/projects.json + dest: projects.json + + - name: upload script + ansible.builtin.copy: + src: clean_timelines_json.py + dest: clean_timelines_json.py + + - name: run script + ansible.builtin.command: python3 clean_timelines_json.py --act + become_user: safekeeper + become: True + register: hello + + - debug: var=hello + + - name: remove projects.json + ansible.builtin.file: + path: projects.json + state: absent + tags: + - rm_file