From 2f13fa225f655cc86b1e7f4b07e412a48adff9ef Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Mon, 6 May 2024 11:49:11 +0530 Subject: [PATCH] Chore (python): Better retry loop logging when embedding api fails (#1267) https://github.com/lancedb/lancedb/issues/1266#event-12703166915 This happens because openai API errors out with None values. The current log level didn't really print out the msg on screen. Changed the log level to warning, which better suits this case. Also, retry loop can be disabled by setting `max_retries=0` (I'm not sure if we should also set this as the default behaviour as hitting api rate is quite common when ingesting large corpus) ``` func = get_registry().get("openai").create(max_retries=0) ```` --- python/python/lancedb/embeddings/utils.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python/python/lancedb/embeddings/utils.py b/python/python/lancedb/embeddings/utils.py index 3c0923c1..813631ca 100644 --- a/python/python/lancedb/embeddings/utils.py +++ b/python/python/lancedb/embeddings/utils.py @@ -255,7 +255,13 @@ def retry_with_exponential_backoff( ) delay *= exponential_base * (1 + jitter * random.random()) - logging.info("Retrying in %s seconds...", delay) + logging.warning( + "Error occurred: %s \n Retrying in %s seconds (retry %s of %s) \n", + e, + delay, + num_retries, + max_retries, + ) time.sleep(delay) return wrapper