[evolution-ews/gnome-41] I#174 - Silently retry on I/O errors (like 'Connection terminated unexpectedly')
- From: Milan Crha <mcrha src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [evolution-ews/gnome-41] I#174 - Silently retry on I/O errors (like 'Connection terminated unexpectedly')
- Date: Fri, 7 Jan 2022 09:27:02 +0000 (UTC)
commit d54b8f319f83e49d8b1729347c1f6bb70d4f191e
Author: Milan Crha <mcrha redhat com>
Date: Fri Jan 7 10:25:34 2022 +0100
I#174 - Silently retry on I/O errors (like 'Connection terminated unexpectedly')
Closes https://gitlab.gnome.org/GNOME/evolution-ews/-/issues/174
src/EWS/common/e-ews-connection-utils.c | 4 ++--
src/EWS/common/e-ews-connection.c | 24 +++++++++++++++++++-----
src/EWS/common/e-ews-notification.c | 8 ++++----
src/EWS/common/e-soap-response.c | 2 +-
src/Microsoft365/common/e-m365-connection.c | 23 +++++++++++++++++------
5 files changed, 43 insertions(+), 18 deletions(-)
---
diff --git a/src/EWS/common/e-ews-connection-utils.c b/src/EWS/common/e-ews-connection-utils.c
index a9616d59..ba548774 100644
--- a/src/EWS/common/e-ews-connection-utils.c
+++ b/src/EWS/common/e-ews-connection-utils.c
@@ -463,7 +463,7 @@ e_ews_connection_utils_authenticate (EEwsConnection *cnc,
ews_connection_utils_setup_bearer_auth (cnc, session, msg, TRUE, E_SOUP_AUTH_BEARER (auth),
NULL, &local_error);
if (local_error)
- soup_message_set_status_full (msg, SOUP_STATUS_IO_ERROR, local_error->message);
+ soup_message_set_status_full (msg, SOUP_STATUS_MALFORMED, local_error->message);
g_object_unref (using_bearer_auth);
g_clear_error (&local_error);
@@ -477,7 +477,7 @@ e_ews_connection_utils_authenticate (EEwsConnection *cnc,
e_ews_connection_utils_expired_password_to_error (service_url, &local_error);
if (local_error)
- soup_message_set_status_full (msg, SOUP_STATUS_IO_ERROR, local_error->message);
+ soup_message_set_status_full (msg, SOUP_STATUS_MALFORMED, local_error->message);
g_clear_error (&local_error);
g_free (service_url);
diff --git a/src/EWS/common/e-ews-connection.c b/src/EWS/common/e-ews-connection.c
index 916aab15..ea0ccca9 100644
--- a/src/EWS/common/e-ews-connection.c
+++ b/src/EWS/common/e-ews-connection.c
@@ -30,6 +30,8 @@
#define d(x) x
+#define EWS_RETRY_IO_ERROR_SECONDS 3
+
/* A chunk size limit when moving items in chunks. */
#define EWS_MOVE_ITEMS_CHUNK_SIZE 500
@@ -153,6 +155,8 @@ struct _EwsNode {
GCancellable *cancellable;
gulong cancel_handler_id;
+
+ gboolean retrying_after_network_error;
};
struct _EwsUrls {
@@ -922,7 +926,7 @@ ews_response_cb (SoupSession *session,
gpointer data)
{
EwsNode *enode = (EwsNode *) data;
- ESoapResponse *response;
+ ESoapResponse *response = NULL;
ESoapParameter *param;
const gchar *persistent_auth;
gint log_level;
@@ -978,6 +982,11 @@ ews_response_cb (SoupSession *session,
EWS_CONNECTION_ERROR_AUTHENTICATION_FAILED,
_("Authentication failed"));
goto exit;
+ } else if (!enode->retrying_after_network_error &&
+ msg->status_code == SOUP_STATUS_IO_ERROR) {
+ wait_ms = EWS_RETRY_IO_ERROR_SECONDS * 1000;
+ enode->retrying_after_network_error = TRUE;
+ goto retrylbl;
} else if (msg->status_code == SOUP_STATUS_CANT_RESOLVE ||
msg->status_code == SOUP_STATUS_CANT_RESOLVE_PROXY ||
msg->status_code == SOUP_STATUS_CANT_CONNECT ||
@@ -1015,7 +1024,10 @@ ews_response_cb (SoupSession *session,
e_soap_response_dump_response (response, stdout);
}
- param = e_soap_response_get_first_parameter_by_name (response, "detail", NULL);
+ if (!wait_ms && e_ews_connection_get_backoff_enabled (enode->cnc))
+ param = e_soap_response_get_first_parameter_by_name (response, "detail", NULL);
+ else
+ param = NULL;
if (param)
param = e_soap_parameter_get_first_child_by_name (param, "ResponseCode");
if (param) {
@@ -1042,7 +1054,8 @@ ews_response_cb (SoupSession *session,
g_free (value);
}
- if (wait_ms > 0 && e_ews_connection_get_backoff_enabled (enode->cnc)) {
+ retrylbl:
+ if (wait_ms > 0) {
GCancellable *cancellable = enode->cancellable;
EFlag *flag;
@@ -1086,7 +1099,7 @@ ews_response_cb (SoupSession *session,
e_flag_free (flag);
- g_object_unref (response);
+ g_clear_object (&response);
if (g_cancellable_is_cancelled (cancellable) ||
msg->status_code == SOUP_STATUS_CANCELLED) {
@@ -1101,6 +1114,7 @@ ews_response_cb (SoupSession *session,
new_node->cb = enode->cb;
new_node->cnc = enode->cnc;
new_node->simple = enode->simple;
+ new_node->retrying_after_network_error = enode->retrying_after_network_error;
enode->simple = NULL;
@@ -1123,7 +1137,7 @@ ews_response_cb (SoupSession *session,
if (enode->cb != NULL)
enode->cb (response, enode->simple);
- g_object_unref (response);
+ g_clear_object (&response);
exit:
if (enode->simple)
diff --git a/src/EWS/common/e-ews-notification.c b/src/EWS/common/e-ews-notification.c
index 28325298..e0ca7fc0 100644
--- a/src/EWS/common/e-ews-notification.c
+++ b/src/EWS/common/e-ews-notification.c
@@ -877,16 +877,16 @@ e_ews_notification_get_events_sync (EEwsNotification *notification,
if (e_ews_debug_get_log_level () <= 3)
soup_message_body_set_accumulate (SOUP_MESSAGE (msg)->response_body, FALSE);
- handler_id = g_signal_connect (
- SOUP_MESSAGE (msg), "got-chunk",
- G_CALLBACK (ews_notification_soup_got_chunk), notification);
-
if (!e_ews_connection_utils_prepare_message (cnc, notification->priv->soup_session, SOUP_MESSAGE
(msg), notification->priv->cancellable)) {
g_object_unref (msg);
g_object_unref (cnc);
return FALSE;
}
+ handler_id = g_signal_connect (
+ SOUP_MESSAGE (msg), "got-chunk",
+ G_CALLBACK (ews_notification_soup_got_chunk), notification);
+
/* Unref it early, thus it doesn't keep the connection alive after all backends are freed */
g_object_unref (cnc);
diff --git a/src/EWS/common/e-soap-response.c b/src/EWS/common/e-soap-response.c
index 45de3069..e503a142 100644
--- a/src/EWS/common/e-soap-response.c
+++ b/src/EWS/common/e-soap-response.c
@@ -562,7 +562,7 @@ e_soap_response_get_first_parameter_by_name (ESoapResponse *response,
g_set_error (
error,
- SOUP_HTTP_ERROR, SOUP_STATUS_IO_ERROR,
+ SOUP_HTTP_ERROR, SOUP_STATUS_MALFORMED,
"%s", (string != NULL) ? string :
"<faultstring> in SOAP response");
diff --git a/src/Microsoft365/common/e-m365-connection.c b/src/Microsoft365/common/e-m365-connection.c
index e81d06ed..b32ca8fa 100644
--- a/src/Microsoft365/common/e-m365-connection.c
+++ b/src/Microsoft365/common/e-m365-connection.c
@@ -19,6 +19,7 @@
#define LOCK(x) g_rec_mutex_lock (&(x->priv->property_lock))
#define UNLOCK(x) g_rec_mutex_unlock (&(x->priv->property_lock))
+#define M365_RETRY_IO_ERROR_SECONDS 3
#define X_EVO_M365_DATA "X-EVO-M365-DATA"
typedef enum _CSMFlags {
@@ -274,7 +275,7 @@ m365_connection_authenticate (SoupSession *session,
m365_connection_utils_setup_bearer_auth (cnc, session, msg, TRUE, E_SOUP_AUTH_BEARER (auth), NULL,
&local_error);
if (local_error)
- soup_message_set_status_full (msg, SOUP_STATUS_IO_ERROR, local_error->message);
+ soup_message_set_status_full (msg, SOUP_STATUS_MALFORMED, local_error->message);
g_object_unref (using_bearer_auth);
g_clear_error (&local_error);
@@ -1139,6 +1140,7 @@ m365_connection_send_request_sync (EM365Connection *cnc,
SoupSession *soup_session;
gint need_retry_seconds = 5;
gboolean success = FALSE, need_retry = TRUE;
+ gboolean did_io_error_retry = FALSE;
g_return_val_if_fail (E_IS_M365_CONNECTION (cnc), FALSE);
g_return_val_if_fail (SOUP_IS_MESSAGE (message), FALSE);
@@ -1237,10 +1239,12 @@ m365_connection_send_request_sync (EM365Connection *cnc,
if (success && m365_log_enabled ())
input_stream = e_soup_logger_attach (message, input_stream);
- /* Throttling - https://docs.microsoft.com/en-us/graph/throttling */
- if (message->status_code == 429 ||
+ if ((!did_io_error_retry && message->status_code == SOUP_STATUS_IO_ERROR) ||
+ /* Throttling - https://docs.microsoft.com/en-us/graph/throttling */
+ message->status_code == 429 ||
/*
https://docs.microsoft.com/en-us/graph/best-practices-concept#handling-expected-errors */
message->status_code == SOUP_STATUS_SERVICE_UNAVAILABLE) {
+ did_io_error_retry = did_io_error_retry || message->status_code ==
SOUP_STATUS_IO_ERROR;
need_retry = TRUE;
} else if (message->status_code == SOUP_STATUS_SSL_FAILED) {
m365_connection_extract_ssl_data (cnc, message);
@@ -1254,6 +1258,8 @@ m365_connection_send_request_sync (EM365Connection *cnc,
if (retry_after_str && *retry_after_str)
retry_after = g_ascii_strtoll (retry_after_str, NULL, 10);
+ else if (message->status_code == SOUP_STATUS_IO_ERROR)
+ retry_after = M365_RETRY_IO_ERROR_SECONDS;
else
retry_after = 0;
@@ -1968,7 +1974,7 @@ e_m365_connection_batch_request_internal_sync (EM365Connection *cnc,
if (!submessage)
continue;
- submessage->status_code = SOUP_STATUS_IO_ERROR;
+ submessage->status_code = SOUP_STATUS_MALFORMED;
suri = soup_message_get_uri (submessage);
uri = suri ? soup_uri_to_string (suri, TRUE) : NULL;
@@ -2086,6 +2092,7 @@ e_m365_connection_batch_request_sync (EM365Connection *cnc,
GPtrArray *use_requests;
gint need_retry_seconds = 5;
gboolean success, need_retry = TRUE;
+ gboolean did_io_error_retry = FALSE;
g_return_val_if_fail (E_IS_M365_CONNECTION (cnc), FALSE);
g_return_val_if_fail (requests != NULL, FALSE);
@@ -2110,13 +2117,15 @@ e_m365_connection_batch_request_sync (EM365Connection *cnc,
if (!message)
continue;
- /* Throttling - https://docs.microsoft.com/en-us/graph/throttling */
- if (message->status_code == 429 ||
+ if ((!did_io_error_retry && message->status_code == SOUP_STATUS_IO_ERROR) ||
+ /* Throttling - https://docs.microsoft.com/en-us/graph/throttling */
+ message->status_code == 429 ||
/*
https://docs.microsoft.com/en-us/graph/best-practices-concept#handling-expected-errors */
message->status_code == SOUP_STATUS_SERVICE_UNAVAILABLE) {
const gchar *retry_after_str;
gint64 retry_after;
+ did_io_error_retry = did_io_error_retry || message->status_code ==
SOUP_STATUS_IO_ERROR;
need_retry = TRUE;
if (!new_requests)
@@ -2128,6 +2137,8 @@ e_m365_connection_batch_request_sync (EM365Connection *cnc,
if (retry_after_str && *retry_after_str)
retry_after = g_ascii_strtoll (retry_after_str, NULL, 10);
+ else if (message->status_code == SOUP_STATUS_IO_ERROR)
+ retry_after = M365_RETRY_IO_ERROR_SECONDS;
else
retry_after = 0;
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
