[pygobject/wip/gio-async-awaitable-return: 1/4] async: Add a new async type that is an awaitable for a _finish call

[Benjamin Berg <bberg src gnome org>]

commit f4369585c29c926067df3aa56b8150abff756a4f
Author: Benjamin Berg <bberg redhat com>
Date:   Fri Nov 13 00:19:47 2020 +0100

    async: Add a new async type that is an awaitable for a _finish call

 gi/gimodule.c    |   3 +
 gi/meson.build   |   1 +
 gi/pygi-async.c  | 554 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 gi/pygi-async.h  |  70 +++++++
 tests/test_gi.py |   3 +
 5 files changed, 631 insertions(+)
---
diff --git a/gi/gimodule.c b/gi/gimodule.c
index 2d1dfe20..1501bd47 100644
--- a/gi/gimodule.c
+++ b/gi/gimodule.c
@@ -34,6 +34,7 @@
 #include "pygi-error.h"
 #include "pygi-foreign.h"
 #include "pygi-resulttuple.h"
+#include "pygi-async.h"
 #include "pygi-source.h"
 #include "pygi-ccallback.h"
 #include "pygi-closure.h"
@@ -2557,6 +2558,8 @@ PYGI_MODINIT_FUNC PyInit__gi(void) {
         return NULL;
     if (pygi_resulttuple_register_types (module) < 0)
         return NULL;
+    if (pygi_async_register_types (module) < 0)
+        return NULL;
 
     if (pygi_spawn_register_types (module_dict) < 0)
         return NULL;
diff --git a/gi/meson.build b/gi/meson.build
index 8edf8328..7eb3311e 100644
--- a/gi/meson.build
+++ b/gi/meson.build
@@ -17,6 +17,7 @@ sources = [
   'pygi-source.c',
   'pygi-argument.c',
   'pygi-resulttuple.c',
+  'pygi-async.c',
   'pygi-type.c',
   'pygi-boxed.c',
   'pygi-closure.c',
diff --git a/gi/pygi-async.c b/gi/pygi-async.c
new file mode 100644
index 00000000..48191563
--- /dev/null
+++ b/gi/pygi-async.c
@@ -0,0 +1,554 @@
+/* -*- Mode: C; c-basic-offset: 4 -*-
+ * vim: tabstop=4 shiftwidth=4 expandtab
+ *
+ * Copyright (C) 2015 Christoph Reiter <reiter christoph gmail com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <Python.h>
+#include <structmember.h>
+#include <glib.h>
+#include "pygobject-object.h"
+#include "pygi-async.h"
+#include "pygi-util.h"
+#include "pygi-info.h"
+#include "pygi-invoke.h"
+
+
+static PyObject *asyncio_InvalidStateError;
+static PyObject *asyncio_get_event_loop;
+
+/* This is never instantiated. */
+PYGI_DEFINE_TYPE ("gi._gi.AsyncBase", PyGIAsyncBase_Type, PyObject)
+
+/**
+ * Async.__repr__() implementation.
+ * Takes the _Async.__repr_format format string and applies the finish function
+ * info to it.
+ */
+static PyObject*
+async_repr(PyObject *self) {
+    PyObject *string;
+
+    string =PyUnicode_FromString ("gi.Async(finish_func=blub)");
+
+    return string;
+}
+
+/**
+ * async_cancel:
+ *
+ * Cancel the asynchronous operation.
+ */
+static PyObject *
+async_cancel(PyGIAsync *self) {
+
+    return PyObject_CallMethodObjArgs (self->cancellable, NULL);
+}
+
+static PyObject *
+async_done(PyGIAsync *self) {
+
+    return PyBool_FromLong (self->result || self->exception);
+}
+
+static PyObject *
+async_result(PyGIAsync *self) {
+
+    if (!self->result && !self->exception) {
+        PyErr_SetString(asyncio_InvalidStateError, "Async task is still running!");
+        return NULL;
+    }
+
+    if (self->result) {
+        Py_INCREF (self->result);
+        return self->result;
+    } else {
+        PyErr_SetObject(PyExceptionInstance_Class(self->exception), self->exception);
+        return NULL;
+    }
+}
+
+static PyObject *
+async_exception(PyGIAsync *self) {
+
+    PyObject *res;
+
+    if (!self->result && !self->exception) {
+        PyErr_SetString(asyncio_InvalidStateError, "Async task is still running!");
+        return NULL;
+    }
+
+    if (self->exception)
+        res =self->exception;
+    else
+        res = Py_None;
+
+    Py_INCREF (res);
+    return res;
+}
+
+static PyObject*
+call_soon (PyGIAsync *self, PyGIAsyncCallback *cb)
+{
+    PyObject *call_soon;
+    PyObject *args, *kwargs;
+    PyObject *ret;
+
+    call_soon = PyObject_GetAttrString(self->loop, "call_soon");
+    if (!call_soon)
+        return NULL;
+
+    args = Py_BuildValue ("(OO)", cb->func, self);
+#if PY_VERSION_HEX >= 0x03070000
+    kwargs = PyDict_New ();
+    PyDict_SetItemString (kwargs, "context", cb->context);
+#endif
+    ret = PyObject_Call (call_soon, args, kwargs);
+
+    Py_CLEAR (args);
+    Py_CLEAR (kwargs);
+    Py_CLEAR (call_soon);
+
+    return ret;
+}
+
+static PyObject*
+async_add_done_callback (PyGIAsync *self, PyObject *args, PyObject *kwargs)
+{
+    static char * kwlist[] = {"", "context", NULL};
+    PyGIAsyncCallback callback = { NULL };
+
+#if PY_VERSION_HEX >= 0x03070000
+    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|$O:add_done_callback", kwlist,
+                                     &callback.func, &callback.context))
+        return NULL;
+
+    Py_INCREF(callback.func);
+    if (callback.context == NULL)
+        callback.context = PyContext_CopyCurrent ();
+    else
+        Py_INCREF(callback.context);
+#else
+    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O:add_done_callback", NULL,
+                                     &callback.func))
+        return NULL;
+#endif
+
+    /* Note that we don't need to copy the current context in this case. */
+    if (self->result || self->exception) {
+        PyObject *res = call_soon (self, &callback);
+
+        Py_DECREF(callback.func);
+#if PY_VERSION_HEX >= 0x03070000
+        Py_DECREF(callback.context);
+#endif
+        if (res) {
+            Py_DECREF(res);
+            Py_RETURN_NONE;
+        } else {
+            return NULL;
+        }
+    }
+
+    if (!self->callbacks)
+        self->callbacks = g_array_new (TRUE, TRUE, sizeof (PyGIAsyncCallback));
+
+    g_array_append_val (self->callbacks, callback);
+
+    Py_RETURN_NONE;
+}
+
+static PyObject*
+async_remove_done_callback (PyGIAsync *self, PyObject *fn)
+{
+    guint i = 0;
+    gssize removed = 0;
+
+    while (self->callbacks && i < self->callbacks->len) {
+        PyGIAsyncCallback *cb = &g_array_index (self->callbacks, PyGIAsyncCallback, i);
+
+        if (PyObject_RichCompareBool (cb->func, fn, Py_EQ) == 1) {
+            Py_DECREF(cb->func);
+#if PY_VERSION_HEX >= 0x03070000
+            Py_DECREF(cb->context);
+#endif
+
+            removed += 1;
+            g_array_remove_index (self->callbacks, i);
+        } else {
+            i += 1;
+        }
+    }
+
+    return PyLong_FromSsize_t(removed);
+}
+
+static int
+async_init(PyGIAsync *self, PyObject *args, PyObject *kwargs)
+{
+    static char *kwlist[] = { NULL };
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "", kwlist, NULL))
+        return -1;
+
+    /* We currently do not support overriding the loop manual. So always set
+     * the current thread-local main loop.
+     */
+    self->loop = PyObject_CallObject (asyncio_get_event_loop, NULL);
+    if (!self->loop)
+        return -1;
+
+    return 0;
+}
+
+static PyMethodDef async_methods[] = {
+    {"cancel", (PyCFunction)async_cancel, METH_NOARGS},
+    {"done", (PyCFunction)async_done, METH_NOARGS},
+    {"result", (PyCFunction)async_result, METH_NOARGS},
+    {"exception", (PyCFunction)async_exception, METH_NOARGS},
+    {"add_done_callback", (PyCFunction)async_add_done_callback, METH_VARARGS | METH_KEYWORDS},
+    {"remove_done_callback", (PyCFunction)async_remove_done_callback, METH_O},
+    {NULL, NULL, 0},
+};
+
+static PyObject *
+async_await (PyGIAsync *self)
+{
+    /* We return ourselves as iterator. This is legal in principle, but we
+     * don't stop iterating after one item and just continue indefinately,
+     * meaning that certain errors cannot be caught!
+     */
+
+    if (!self->result && !self->exception)
+        self->_asyncio_future_blocking = TRUE;
+
+    Py_INCREF (self);
+    return (PyObject *) self;
+}
+
+static PyObject *
+async_iternext (PyGIAsync *self)
+{
+
+    /* Return ourselves if iteration needs to continue. */
+    if (!self->result && !self->exception) {
+        Py_INCREF (self);
+        return (PyObject *) self;
+    }
+
+    if (self->exception) {
+        PyErr_SetObject(PyExceptionInstance_Class(self->exception), self->exception);
+        return NULL;
+    } else {
+        PyErr_SetObject(PyExc_StopIteration, self->result);
+        return NULL;
+    }
+}
+
+static PyAsyncMethods async_async_methods = {
+    .am_await = (unaryfunc) async_await,
+};
+
+static void
+async_dealloc(PyGIAsync *self)
+{
+    Py_CLEAR(self->loop);
+    if (FALSE)
+        Py_CLEAR(self->cancellable);
+    if (self->result)
+        Py_CLEAR(self->result);
+    if (self->exception)
+        Py_CLEAR(self->exception);
+
+    /* XXX: This should never happen! */
+    if (self->callbacks)
+        g_array_free (self->callbacks, TRUE);
+
+    PyObject_DEL(self);
+}
+
+
+void
+pygi_async_finish_cb (GObject *source_object, gpointer res, PyGIAsync *self)
+{
+    PyGIAsyncType *type = (PyGIAsyncType *) PyObject_Type((PyObject *) self);
+    PyGILState_STATE py_state;
+    PyObject *source_pyobj, *res_pyobj, *args;
+    PyObject *ret;
+    guint i;
+
+    /* Lock the GIL as we are coming into this code without the lock and we
+      may be executing python code */
+    py_state = PyGILState_Ensure ();
+
+    /* We might still be called at shutdown time. */
+    if (!Py_IsInitialized()) {
+        PyGILState_Release (py_state);
+        return;
+    }
+
+    res_pyobj = pygobject_new_full (res, FALSE, NULL);
+    if (source_object) {
+        source_pyobj = pygobject_new_full (source_object, FALSE, NULL);
+        args = Py_BuildValue ("(OO)", source_pyobj, res_pyobj);
+    } else {
+        source_pyobj = NULL;
+        args = Py_BuildValue ("(O)", res_pyobj);
+    }
+    ret = _wrap_g_callable_info_invoke ((PyGIBaseInfo *) type->async_finish, args, NULL);
+    Py_XDECREF (res_pyobj);
+    Py_XDECREF (source_pyobj);
+    Py_XDECREF (args);
+
+    if (PyErr_Occurred ()) {
+        PyObject *exc = NULL, *value = NULL, *traceback = NULL;
+        PyObject *exc_val;
+
+        PyErr_Fetch (&exc, &value, &traceback);
+
+        Py_XDECREF (ret);
+
+        if (!value)
+            exc_val = PyObject_CallFunction(exc, "");
+        else
+            exc_val = PyObject_CallFunction(exc, "O", value);
+
+        if (exc_val == NULL)
+            exc_val = PyObject_CallFunction(PyExc_TypeError, "Invalid exception from _finish function 
call!");
+
+        g_assert (exc_val);
+        self->exception = exc_val;
+
+        Py_XDECREF (exc);
+        Py_XDECREF (value);
+        Py_XDECREF (traceback);
+        Py_XDECREF (ret);
+    } else {
+        self->result = ret;
+    }
+
+    /* TODO: Call the notifiers */
+    for (i = 0; self->callbacks && i < self->callbacks->len; i++) {
+        PyGIAsyncCallback *cb = &g_array_index (self->callbacks, PyGIAsyncCallback, i);
+        /* We stop calling anything after the first exception, but still clear
+         * the internal state as if we did.
+         * This matches the pure python implementation of Future.
+         */
+        if (!PyErr_Occurred ()) {
+            ret = call_soon (self, cb);
+            if (!ret)
+                PyErr_PrintEx (FALSE);
+            else
+                Py_DECREF(ret);
+        }
+
+        Py_DECREF(cb->func);
+#if PY_VERSION_HEX >= 0x03070000
+        Py_DECREF(cb->context);
+#endif
+    }
+    if (self->callbacks)
+        g_array_free(self->callbacks, TRUE);
+    self->callbacks = NULL;
+
+    Py_DECREF(self);
+    PyGILState_Release (py_state);
+}
+
+/**
+ * PyGIAsyncType_type.tp_getattro implementation.
+ * Exists solely to return the finish_func of the class.
+ */
+static PyObject*
+async_type_getattro(PyObject *self, PyObject *name) {
+
+    if (PyUnicode_Check (name) && PyUnicode_CompareWithASCIIString (name, "finish_func") == 0) {
+        PyGIAsyncType *type = (PyGIAsyncType *) PyObject_Type(self);
+
+        Py_INCREF (type->async_finish);
+        return (PyObject *) type->async_finish;
+    }
+
+    return PyGIAsyncBase_Type.tp_getattro (self, name);
+}
+
+static void
+async_type_dealloc(PyGIAsyncType *self)
+{
+    Py_CLEAR(self->async_finish);
+
+    /* Chain up to normal type deallocator. */
+    PyType_Type.tp_dealloc((PyObject *) self);
+}
+
+/**
+ * pygi_async_new_type:
+ * @async_finish: A #GIFunctionInfo to wrap that is used to finish.
+ *
+ * Creates a result type for a pending asynchronous operation of a specific
+ * async function. This works by finding the corresponding finish function,
+ * and checking that it is compatible. If it is, a new type will be returned
+ * that can be cached for future use.
+ *
+ * Returns: A new PyGIAsyncType PyTypeObject of type PyGIAsyncType_Type
+ *    or %NULL in case of an error.
+ */
+PyTypeObject*
+pygi_async_new_type(GIFunctionInfo *async_finish) {
+    PyGIAsyncType *new;
+    PyObject *new_type;
+    static struct PyMemberDef members[] = {
+        {
+            "_asyncio_future_blocking",
+            T_BOOL,
+            offsetof(PyGIAsync, _asyncio_future_blocking),
+            0,
+            NULL
+        },
+        {
+            "_loop",
+            T_OBJECT,
+            offsetof(PyGIAsync, loop),
+            READONLY,
+            NULL
+        },
+        {
+            "cancellable",
+            T_OBJECT,
+            offsetof(PyGIAsync, cancellable),
+            READONLY,
+            "The Gio.Cancellable associated with the task."
+        },
+        { NULL, }
+    };
+
+    PyObject *class_dict, *slots, *new_type_args;
+
+    class_dict = PyDict_New ();
+
+    /* To save some memory don't use an instance dict */
+    slots = PyTuple_New (0);
+    PyDict_SetItemString (class_dict, "__slots__", slots);
+    Py_DECREF (slots);
+
+    g_assert (async_finish != NULL);
+
+    new_type_args = Py_BuildValue ("s(O)O", "_Async", &PyGIAsyncBase_Type, class_dict);
+
+    new_type = PyType_Type.tp_new (&PyType_Type, new_type_args, NULL);
+    Py_DECREF (class_dict);
+    if (!new_type)
+        return NULL;
+
+    new = (PyGIAsyncType *) new_type;
+    new->ht_object.ht_type.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE;
+    new->ht_object.ht_type.tp_dealloc = (destructor)async_dealloc;
+    new->ht_object.ht_type.tp_getattro = async_type_getattro;
+    new->ht_object.ht_type.tp_members = members;
+    new->ht_object.ht_type.tp_basicsize = sizeof(PyGIAsync);
+    new->ht_object.ht_type.tp_itemsize = 0;
+
+    new->async_finish = (PyGICallableInfo*) _pygi_info_new ((GIBaseInfo *) async_finish);
+
+    if (PyType_Ready ((PyTypeObject *) new_type) < 0) {
+        Py_DECREF (new_type);
+        return NULL;
+    }
+
+    return (PyTypeObject *) new_type;
+}
+
+
+/**
+ * pygi_async_new:
+ * @subclass: A PyGIAsyncType_Type subclass which will be the type of the
+ *    returned instance.
+ *
+ * Return a new async instance.
+ *
+ * Returns: An instance of @subclass or %NULL on error.
+ */
+PyObject *
+pygi_async_new(PyTypeObject *subclass) {
+
+    PyObject *res;
+    PyObject *args;
+
+    res = subclass->tp_alloc (subclass, 0);
+
+    if (res) {
+        args = PyTuple_New (0);
+        subclass->tp_init (res, args, NULL);
+        Py_DECREF(args);
+    }
+
+    return res;
+}
+
+/**
+ * pygi_async_register_types:
+ * @module: A Python modules to which Async gets added to.
+ *
+ * Initializes the Async class and adds it to the passed @module.
+ *
+ * Returns: -1 on error, 0 on success.
+ */
+int pygi_async_register_types(PyObject *module) {
+    PyObject *asyncio_exceptions = NULL;
+    PyObject *asyncio_events = NULL;
+
+    PyGIAsyncBase_Type.tp_dealloc = (destructor)async_type_dealloc;
+    PyGIAsyncBase_Type.tp_repr = (reprfunc)async_repr;
+    PyGIAsyncBase_Type.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE;
+    PyGIAsyncBase_Type.tp_methods = async_methods;
+    PyGIAsyncBase_Type.tp_as_async = &async_async_methods;
+    PyGIAsyncBase_Type.tp_iternext = (iternextfunc) &async_iternext;
+    PyGIAsyncBase_Type.tp_init = (initproc)async_init;
+
+    if (PyType_Ready (&PyGIAsyncBase_Type) < 0)
+        return -1;
+
+    Py_INCREF (&PyGIAsyncBase_Type);
+    if (PyModule_AddObject (module, "AsyncBase",
+                            (PyObject *)&PyGIAsyncBase_Type) < 0) {
+        Py_DECREF (&PyGIAsyncBase_Type);
+        return -1;
+    }
+
+    asyncio_exceptions = PyImport_ImportModule("asyncio.exceptions");
+    if (module == NULL) {
+        goto fail;
+    }
+    asyncio_InvalidStateError = PyObject_GetAttrString(asyncio_exceptions, "InvalidStateError");
+    if (asyncio_InvalidStateError == NULL)
+        goto fail;
+
+    asyncio_events = PyImport_ImportModule("asyncio.events");
+    if (module == NULL) {
+        goto fail;
+    }
+    asyncio_get_event_loop = PyObject_GetAttrString(asyncio_events, "get_event_loop");
+    if (asyncio_get_event_loop == NULL)
+        goto fail;
+
+    Py_CLEAR(asyncio_exceptions);
+    Py_CLEAR(asyncio_events);
+    return 0;
+
+fail:
+    Py_CLEAR(asyncio_events);
+    return -1;
+}
diff --git a/gi/pygi-async.h b/gi/pygi-async.h
new file mode 100644
index 00000000..87c4e59f
--- /dev/null
+++ b/gi/pygi-async.h
@@ -0,0 +1,70 @@
+/* -*- Mode: C; c-basic-offset: 4 -*-
+ * vim: tabstop=4 shiftwidth=4 expandtab
+ *
+ * Copyright (C) 2015 Christoph Reiter <reiter christoph gmail com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __PYGI_ASYNC_H__
+#define __PYGI_ASYNC_H__
+
+#include "Python.h"
+
+#include "pygi-info.h"
+#include "pygi-cache.h"
+
+typedef struct {
+    PyHeapTypeObject ht_object;
+
+    /* We keep the async_finish information around solely to provide it as
+     * a read only member.
+     */
+    PyGICallableInfo *async_finish;
+} PyGIAsyncType;
+
+typedef struct {
+    PyObject *func;
+#if PY_VERSION_HEX >= 0x03070000
+    PyObject *context;
+#endif
+} PyGIAsyncCallback;
+
+typedef struct {
+    PyObject object;
+
+    /* Everything for the instance, async_finish is kept in the class. */
+    PyObject *loop;
+    PyObject *cancellable;
+    int _asyncio_future_blocking;
+    PyObject *result;
+    PyObject *exception;
+
+    GArray *callbacks;
+} PyGIAsync;
+
+
+int
+pygi_async_register_types (PyObject *d);
+
+void
+pygi_async_finish_cb (GObject *source_object, gpointer res, PyGIAsync *async);
+
+PyTypeObject *
+pygi_async_new_type       (GIFunctionInfo *async_finish);
+
+PyObject*
+pygi_async_new            (PyTypeObject *subclass);
+
+#endif /* __PYGI_ASYNCRESULT_H__ */
diff --git a/tests/test_gi.py b/tests/test_gi.py
index 5e7d0e35..4d058bda 100644
--- a/tests/test_gi.py
+++ b/tests/test_gi.py
@@ -975,6 +975,9 @@ class TestArray(unittest.TestCase):
     def test_array_out(self):
         self.assertEqual([-1, 0, 1, 2], GIMarshallingTests.array_out())
 
+    def test_array_out_rev(self):
+        self.assertEqual([-1, 0, 1, 2], GIMarshallingTests.array_out_rev())
+
     def test_array_out_etc(self):
         self.assertEqual(([-5, 0, 1, 9], 4), GIMarshallingTests.array_out_etc(-5, 9))