Script&playbook for removing deleted timelines listed in json file.

.circleci/ansible/ansible.cfg

@@ -4,7 +4,11 @@ localhost_warning = False
 host_key_checking = False
 timeout = 30
 
+allow_world_readable_tmpfiles=true
+
 [ssh_connection]
 ssh_args   = -F ./ansible.ssh.cfg
-scp_if_ssh = True
+# teleport doesn't support sftp yet https://github.com/gravitational/teleport/issues/7127
+# and scp neither worked for me
+transfer_method = piped
 pipelining = True

.circleci/ansible/ansible.ssh.cfg

@@ -1,3 +1,6 @@
+# Remove this once https://github.com/gravitational/teleport/issues/10918 is fixed
+PubkeyAcceptedAlgorithms +ssh-rsa-cert-v01@openssh.com
+
 Host tele.zenith.tech
     User admin
     Port 3023

.circleci/ansible/clean_timelines.yml

@@ -0,0 +1,34 @@
+- name: cleanup timelines
+  hosts: safekeepers
+  gather_facts: False
+  remote_user: admin
+  tasks:
+    - name: install pip
+      ansible.builtin.apt:
+        name: 
+          - python3-pip
+          - acl
+      become: True
+
+    - name: install psycopg2
+      ansible.builtin.pip:
+        name: psycopg2
+      become_user: safekeeper
+      become: True
+
+    - name: upload script
+      ansible.builtin.copy:
+        src: /home/ars/zenith/cloud/ops/ad_hoc/clean_timelines.py
+        dest: clean_timelines.py
+      tags:
+        - safekeeper
+    
+    - name: run script
+      ansible.builtin.command: python3 clean_timelines.py --really-delete
+      environment:
+        CONSOLE_DB: "{{ CONSOLE_DB }}"
+      become_user: safekeeper
+      become: True
+      register: hello
+    
+    - debug: var=hello

.circleci/ansible/clean_timelines_json.py

@@ -0,0 +1,46 @@
+import os
+import shutil
+import sys
+import time
+import json
+from pathlib import Path
+
+# Dry run is the default, i.e. not do anything
+act = len(sys.argv) == 2 and sys.argv[1] == '--act'
+print('act=', act)
+
+move_to_deleted = True
+
+# result of GET 'https://console.neon.tech/api/v1/admin/projects?order=asc&sort=id&show_deleted=true'
+with open('projects.json') as f:
+    projects = json.load(f)
+
+projects = projects['data']
+deleted_projects = [p for p in projects if p["deleted"]]
+active_projects = [p for p in projects if not p["deleted"]]
+print(f"deleted {len(deleted_projects)}, active {len(active_projects)}, total {len(projects)}")
+# print(deleted_projects)
+
+for project_to_delete in deleted_projects:
+    tenant_id = project_to_delete['tenant']
+    tenant_dir = Path(f"/storage/safekeeper/data/{tenant_id}")
+    timeline_dir = tenant_dir / project_to_delete['timeline_id']
+     
+    if not os.path.exists(timeline_dir):
+        continue
+
+    if move_to_deleted:
+        # move to deleted/
+        tenant_dir_deleted = (Path(f"/storage/safekeeper/data/deleted/") / tenant_id)
+        tenant_dir_deleted.mkdir(parents=True, exist_ok=True)
+        print(f"moving {timeline_dir} to {tenant_dir_deleted}")
+        if act:
+            shutil.move(timeline_dir, tenant_dir_deleted)
+    else:
+        print(f"removing {timeline_dir}")
+        if act:
+            shutil.rmtree(timeline_dir)
+
+    if len(os.listdir(tenant_dir)) == 0:
+        print(f"removing empty tenant directory {tenant_dir}")
+        shutil.rmtree(tenant_dir)

.circleci/ansible/clean_timelines_json.yml

@@ -0,0 +1,36 @@
+- name: remove timelines listed in projects.json
+  hosts: safekeepers
+  gather_facts: False
+  remote_user: admin
+  tasks:
+    # - name: install acl
+    #   ansible.builtin.apt:
+    #     name: 
+    #       - python3-pip
+    #       - acl
+    #   become: True
+
+    - name: upload projects.json
+      ansible.builtin.copy:
+        src: /home/ars/zenith/projects.json
+        dest: projects.json
+
+    - name: upload script
+      ansible.builtin.copy:
+        src: clean_timelines_json.py
+        dest: clean_timelines_json.py
+    
+    - name: run script
+      ansible.builtin.command: python3 clean_timelines_json.py --act
+      become_user: safekeeper
+      become: True
+      register: hello
+    
+    - debug: var=hello
+
+    - name: remove projects.json
+      ansible.builtin.file:
+        path: projects.json
+        state: absent
+      tags:
+        - rm_file