import pathlib from .fixtures import create_collection, upsert_random_points from .utils import * from .assertions import assert_http_ok N_PEERS = 3 COLLECTION_NAME = "test_collection" def test_collection_recovery(tmp_path: pathlib.Path): assert_project_root() peer_urls, peer_dirs, bootstrap_url = start_cluster(tmp_path, N_PEERS) create_collection(peer_urls[0], shard_number=2, replication_factor=2) wait_collection_exists_and_active_on_all_peers(collection_name="test_collection", peer_api_uris=peer_urls) upsert_random_points(peer_urls[0], 100) # Check that collection is not empty info = get_collection_cluster_info(peer_urls[-1], COLLECTION_NAME) for shard in info["local_shards"]: assert shard["points_count"] > 0 # Kill last peer processes.pop().kill() # Delete collection files from disk collection_path = Path(peer_dirs[-1]) / "storage" / "collections" / COLLECTION_NAME assert collection_path.exists() shutil.rmtree(collection_path) # Restart the peer peer_url = start_peer(peer_dirs[-1], "peer_0_restarted.log", bootstrap_url) # Wait until peer is up wait_for_peer_online(peer_url) # Recover Raft state recover_raft_state(peer_url) # Wait for all shards to be active wait_for(all_collection_shards_are_active, peer_url, COLLECTION_NAME) # Check, that the collection is not empty on recovered node info = get_collection_cluster_info(peer_url, COLLECTION_NAME) for shard in info["local_shards"]: assert shard["points_count"] > 0 def recover_raft_state(peer_url): r = requests.post(f"{peer_url}/cluster/recover") return request_result(r) def request_result(resp): assert_http_ok(resp) return resp.json()["result"] def all_collection_shards_are_active(peer_url, collection_name): try: info = get_collection_cluster_info(peer_url, collection_name) except: return False remote_shards = info["remote_shards"] local_shards = info["local_shards"] if len(remote_shards) == 0: return False return all(map(lambda shard: shard["state"] == "Active", local_shards + remote_shards))