technocloud-public/nginx-unit
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Python: split module initialization from WSGI implementation.

Browse Source 
This is required for futher ASGI implementation.
This commit is contained in:
Max Romanov
2020-09-14 13:27:02 +03:00
parent d483aa74e6
commit d94dac091f
4 changed files with 388 additions and 285 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
auto/modules/python
View File

@@ -167,6 +167,7 @@ $echo " + Python module: ${NXT_PYTHON_MODULE}.unit.so"
$echo >> $NXT_MAKEFILE
NXT_PYTHON_MODULE_SRCS=" \
src/python/nxt_python.c \
src/python/nxt_python_wsgi.c \
"

331
src/python/nxt_python.c Normal file
View File

@@ -0,0 +1,331 @@
/*
* Copyright (C) NGINX, Inc.
*/
#include <Python.h>
#include <nxt_main.h>
#include <nxt_router.h>
#include <nxt_unit.h>
#include <python/nxt_python.h>
#include NXT_PYTHON_MOUNTS_H
#if PY_MAJOR_VERSION == 3
#define PyString_FromStringAndSize(str, size) \
PyUnicode_DecodeLatin1((str), (size), "strict")
#else
#define PyUnicode_InternInPlace PyString_InternInPlace
#endif
static nxt_int_t nxt_python_start(nxt_task_t *task,
nxt_process_data_t *data);
static void nxt_python_atexit(void);
static uint32_t compat[] = {
NXT_VERNUM, NXT_DEBUG,
};
NXT_EXPORT nxt_app_module_t nxt_app_module = {
sizeof(compat),
compat,
nxt_string("python"),
PY_VERSION,
nxt_python_mounts,
nxt_nitems(nxt_python_mounts),
NULL,
nxt_python_start,
};
static PyObject *nxt_py_stderr_flush;
PyObject *nxt_py_application;
#if PY_MAJOR_VERSION == 3
static wchar_t *nxt_py_home;
#else
static char *nxt_py_home;
#endif
static nxt_int_t
nxt_python_start(nxt_task_t *task, nxt_process_data_t *data)
{
int rc;
char *nxt_py_module;
size_t len;
PyObject *obj, *pypath, *module;
nxt_unit_ctx_t *unit_ctx;
nxt_unit_init_t python_init;
nxt_common_app_conf_t *app_conf;
nxt_python_app_conf_t *c;
#if PY_MAJOR_VERSION == 3
char *path;
size_t size;
nxt_int_t pep405;
static const char pyvenv[] = "/pyvenv.cfg";
static const char bin_python[] = "/bin/python";
#endif
app_conf = data->app;
c = &app_conf->u.python;
if (c->home != NULL) {
len = nxt_strlen(c->home);
#if PY_MAJOR_VERSION == 3
path = nxt_malloc(len + sizeof(pyvenv));
if (nxt_slow_path(path == NULL)) {
nxt_alert(task, "Failed to allocate memory");
return NXT_ERROR;
}
nxt_memcpy(path, c->home, len);
nxt_memcpy(path + len, pyvenv, sizeof(pyvenv));
pep405 = (access(path, R_OK) == 0);
nxt_free(path);
if (pep405) {
size = (len + sizeof(bin_python)) * sizeof(wchar_t);
} else {
size = (len + 1) * sizeof(wchar_t);
}
nxt_py_home = nxt_malloc(size);
if (nxt_slow_path(nxt_py_home == NULL)) {
nxt_alert(task, "Failed to allocate memory");
return NXT_ERROR;
}
if (pep405) {
mbstowcs(nxt_py_home, c->home, len);
mbstowcs(nxt_py_home + len, bin_python, sizeof(bin_python));
Py_SetProgramName(nxt_py_home);
} else {
mbstowcs(nxt_py_home, c->home, len + 1);
Py_SetPythonHome(nxt_py_home);
}
#else
nxt_py_home = nxt_malloc(len + 1);
if (nxt_slow_path(nxt_py_home == NULL)) {
nxt_alert(task, "Failed to allocate memory");
return NXT_ERROR;
}
nxt_memcpy(nxt_py_home, c->home, len + 1);
Py_SetPythonHome(nxt_py_home);
#endif
}
Py_InitializeEx(0);
module = NULL;
obj = NULL;
obj = PySys_GetObject((char *) "stderr");
if (nxt_slow_path(obj == NULL)) {
nxt_alert(task, "Python failed to get \"sys.stderr\" object");
goto fail;
}
nxt_py_stderr_flush = PyObject_GetAttrString(obj, "flush");
if (nxt_slow_path(nxt_py_stderr_flush == NULL)) {
nxt_alert(task, "Python failed to get \"flush\" attribute of "
"\"sys.stderr\" object");
goto fail;
}
/* obj is a Borrowed reference. */
if (c->path.length > 0) {
obj = PyString_FromStringAndSize((char *) c->path.start,
c->path.length);
if (nxt_slow_path(obj == NULL)) {
nxt_alert(task, "Python failed to create string object \"%V\"",
&c->path);
goto fail;
}
pypath = PySys_GetObject((char *) "path");
if (nxt_slow_path(pypath == NULL)) {
nxt_alert(task, "Python failed to get \"sys.path\" list");
goto fail;
}
if (nxt_slow_path(PyList_Insert(pypath, 0, obj) != 0)) {
nxt_alert(task, "Python failed to insert \"%V\" into \"sys.path\"",
&c->path);
goto fail;
}
Py_DECREF(obj);
}
obj = Py_BuildValue("[s]", "unit");
if (nxt_slow_path(obj == NULL)) {
nxt_alert(task, "Python failed to create the \"sys.argv\" list");
goto fail;
}
if (nxt_slow_path(PySys_SetObject((char *) "argv", obj) != 0)) {
nxt_alert(task, "Python failed to set the \"sys.argv\" list");
goto fail;
}
Py_CLEAR(obj);
nxt_py_module = nxt_alloca(c->module.length + 1);
nxt_memcpy(nxt_py_module, c->module.start, c->module.length);
nxt_py_module[c->module.length] = '\0';
module = PyImport_ImportModule(nxt_py_module);
if (nxt_slow_path(module == NULL)) {
nxt_alert(task, "Python failed to import module \"%s\"", nxt_py_module);
nxt_python_print_exception();
goto fail;
}
obj = PyDict_GetItemString(PyModule_GetDict(module), "application");
if (nxt_slow_path(obj == NULL)) {
nxt_alert(task, "Python failed to get \"application\" "
"from module \"%s\"", nxt_py_module);
goto fail;
}
if (nxt_slow_path(PyCallable_Check(obj) == 0)) {
nxt_alert(task, "\"application\" in module \"%s\" "
"is not a callable object", nxt_py_module);
goto fail;
}
nxt_py_application = obj;
obj = NULL;
Py_INCREF(nxt_py_application);
Py_CLEAR(module);
nxt_unit_default_init(task, &python_init);
python_init.shm_limit = data->app->shm_limit;
rc = nxt_python_wsgi_init(task, &python_init);
if (nxt_slow_path(rc == NXT_ERROR)) {
goto fail;
}
unit_ctx = nxt_unit_init(&python_init);
if (nxt_slow_path(unit_ctx == NULL)) {
goto fail;
}
rc = nxt_python_wsgi_run(unit_ctx);
nxt_unit_done(unit_ctx);
nxt_python_atexit();
exit(rc);
return NXT_OK;
fail:
Py_XDECREF(obj);
Py_XDECREF(module);
nxt_python_atexit();
return NXT_ERROR;
}
nxt_int_t
nxt_python_init_strings(nxt_python_string_t *pstr)
{
PyObject *obj;
while (pstr->string.start != NULL) {
obj = PyString_FromStringAndSize((char *) pstr->string.start,
pstr->string.length);
if (nxt_slow_path(obj == NULL)) {
return NXT_ERROR;
}
PyUnicode_InternInPlace(&obj);
*pstr->object_p = obj;
pstr++;
}
return NXT_OK;
}
void
nxt_python_done_strings(nxt_python_string_t *pstr)
{
PyObject *obj;
while (pstr->string.start != NULL) {
obj = *pstr->object_p;
Py_XDECREF(obj);
*pstr->object_p = NULL;
pstr++;
}
}
static void
nxt_python_atexit(void)
{
nxt_python_wsgi_done();
Py_XDECREF(nxt_py_stderr_flush);
Py_XDECREF(nxt_py_application);
Py_Finalize();
if (nxt_py_home != NULL) {
nxt_free(nxt_py_home);
}
}
void
nxt_python_print_exception(void)
{
PyErr_Print();
#if PY_MAJOR_VERSION == 3
/* The backtrace may be buffered in sys.stderr file object. */
{
PyObject *result;
result = PyObject_CallFunction(nxt_py_stderr_flush, NULL);
if (nxt_slow_path(result == NULL)) {
PyErr_Clear();
return;
}
Py_DECREF(result);
}
#endif
}

31
src/python/nxt_python.h Normal file
View File

@@ -0,0 +1,31 @@
/*
* Copyright (C) NGINX, Inc.
*/
#ifndef _NXT_PYTHON_H_INCLUDED_
#define _NXT_PYTHON_H_INCLUDED_
#include <Python.h>
#include <nxt_unit.h>
extern PyObject *nxt_py_application;
typedef struct {
nxt_str_t string;
PyObject **object_p;
} nxt_python_string_t;
nxt_int_t nxt_python_init_strings(nxt_python_string_t *pstr);
void nxt_python_done_strings(nxt_python_string_t *pstr);
void nxt_python_print_exception(void);
nxt_int_t nxt_python_wsgi_init(nxt_task_t *task, nxt_unit_init_t *init);
int nxt_python_wsgi_run(nxt_unit_ctx_t *ctx);
void nxt_python_wsgi_done(void);
#endif /* _NXT_PYTHON_H_INCLUDED_ */

310
src/python/nxt_python_wsgi.c
View File

@@ -8,17 +8,15 @@
#include <Python.h>
#include <compile.h>
#include <node.h>
#include <nxt_main.h>
#include <nxt_runtime.h>
#include <nxt_router.h>
#include <nxt_unit.h>
#include <nxt_unit_field.h>
#include <nxt_unit_request.h>
#include <nxt_unit_response.h>
#include <python/nxt_python.h>
#include NXT_PYTHON_MOUNTS_H
/*
@@ -68,11 +66,7 @@ typedef struct {
PyObject_HEAD
} nxt_py_error_t;
static nxt_int_t nxt_python_start(nxt_task_t *task,
nxt_process_data_t *data);
static nxt_int_t nxt_python_init_strings(void);
static void nxt_python_request_handler(nxt_unit_request_info_t *req);
static void nxt_python_atexit(void);
static PyObject *nxt_python_create_environ(nxt_task_t *task);
static PyObject *nxt_python_get_environ(nxt_python_run_ctx_t *ctx);
@@ -99,7 +93,6 @@ static PyObject *nxt_py_input_readlines(nxt_py_input_t *self, PyObject *args);
static PyObject *nxt_py_input_iter(PyObject *self);
static PyObject *nxt_py_input_next(PyObject *self);
static void nxt_python_print_exception(void);
static int nxt_python_write(nxt_python_run_ctx_t *ctx, PyObject *bytes);
struct nxt_python_run_ctx_s {
@@ -109,22 +102,6 @@ struct nxt_python_run_ctx_s {
nxt_unit_request_info_t *req;
};
static uint32_t compat[] = {
NXT_VERNUM, NXT_DEBUG,
};
NXT_EXPORT nxt_app_module_t nxt_app_module = {
sizeof(compat),
compat,
nxt_string("python"),
PY_VERSION,
nxt_python_mounts,
nxt_nitems(nxt_python_mounts),
NULL,
nxt_python_start,
};
static PyMethodDef nxt_py_start_resp_method[] = {
{"unit_start_response", nxt_py_start_resp, METH_VARARGS, ""}
@@ -158,22 +135,13 @@ static PyTypeObject nxt_py_input_type = {
};
static PyObject *nxt_py_stderr_flush;
static PyObject *nxt_py_application;
static PyObject *nxt_py_start_resp_obj;
static PyObject *nxt_py_write_obj;
static PyObject *nxt_py_environ_ptyp;
#if PY_MAJOR_VERSION == 3
static wchar_t *nxt_py_home;
#else
static char *nxt_py_home;
#endif
static PyThreadState *nxt_python_thread_state;
static nxt_python_run_ctx_t *nxt_python_run_ctx;
static PyObject *nxt_py_80_str;
static PyObject *nxt_py_close_str;
static PyObject *nxt_py_content_length_str;
@@ -191,11 +159,6 @@ static PyObject *nxt_py_server_port_str;
static PyObject *nxt_py_server_protocol_str;
static PyObject *nxt_py_wsgi_uri_scheme_str;
typedef struct {
nxt_str_t string;
PyObject **object_p;
} nxt_python_string_t;
static nxt_python_string_t nxt_python_strings[] = {
{ nxt_string("80"), &nxt_py_80_str },
{ nxt_string("close"), &nxt_py_close_str },
@@ -213,136 +176,22 @@ static nxt_python_string_t nxt_python_strings[] = {
{ nxt_string("SERVER_PORT"), &nxt_py_server_port_str },
{ nxt_string("SERVER_PROTOCOL"), &nxt_py_server_protocol_str },
{ nxt_string("wsgi.url_scheme"), &nxt_py_wsgi_uri_scheme_str },
{ nxt_null_string, NULL },
};
static nxt_int_t
nxt_python_start(nxt_task_t *task, nxt_process_data_t *data)
nxt_int_t
nxt_python_wsgi_init(nxt_task_t *task, nxt_unit_init_t *init)
{
int rc;
char *nxt_py_module;
size_t len;
PyObject *obj, *pypath, *module;
nxt_unit_ctx_t *unit_ctx;
nxt_unit_init_t python_init;
nxt_common_app_conf_t *app_conf;
nxt_python_app_conf_t *c;
#if PY_MAJOR_VERSION == 3
char *path;
size_t size;
nxt_int_t pep405;
PyObject *obj;
static const char pyvenv[] = "/pyvenv.cfg";
static const char bin_python[] = "/bin/python";
#endif
app_conf = data->app;
c = &app_conf->u.python;
if (c->home != NULL) {
len = nxt_strlen(c->home);
#if PY_MAJOR_VERSION == 3
path = nxt_malloc(len + sizeof(pyvenv));
if (nxt_slow_path(path == NULL)) {
nxt_alert(task, "Failed to allocate memory");
return NXT_ERROR;
}
nxt_memcpy(path, c->home, len);
nxt_memcpy(path + len, pyvenv, sizeof(pyvenv));
pep405 = (access(path, R_OK) == 0);
nxt_free(path);
if (pep405) {
size = (len + sizeof(bin_python)) * sizeof(wchar_t);
} else {
size = (len + 1) * sizeof(wchar_t);
}
nxt_py_home = nxt_malloc(size);
if (nxt_slow_path(nxt_py_home == NULL)) {
nxt_alert(task, "Failed to allocate memory");
return NXT_ERROR;
}
if (pep405) {
mbstowcs(nxt_py_home, c->home, len);
mbstowcs(nxt_py_home + len, bin_python, sizeof(bin_python));
Py_SetProgramName(nxt_py_home);
} else {
mbstowcs(nxt_py_home, c->home, len + 1);
Py_SetPythonHome(nxt_py_home);
}
#else
nxt_py_home = nxt_malloc(len + 1);
if (nxt_slow_path(nxt_py_home == NULL)) {
nxt_alert(task, "Failed to allocate memory");
return NXT_ERROR;
}
nxt_memcpy(nxt_py_home, c->home, len + 1);
Py_SetPythonHome(nxt_py_home);
#endif
}
Py_InitializeEx(0);
module = NULL;
obj = NULL;
if (nxt_slow_path(nxt_python_init_strings() != NXT_OK)) {
if (nxt_slow_path(nxt_python_init_strings(nxt_python_strings) != NXT_OK)) {
nxt_alert(task, "Python failed to init string objects");
goto fail;
}
obj = PySys_GetObject((char *) "stderr");
if (nxt_slow_path(obj == NULL)) {
nxt_alert(task, "Python failed to get \"sys.stderr\" object");
goto fail;
}
nxt_py_stderr_flush = PyObject_GetAttrString(obj, "flush");
if (nxt_slow_path(nxt_py_stderr_flush == NULL)) {
nxt_alert(task, "Python failed to get \"flush\" attribute of "
"\"sys.stderr\" object");
goto fail;
}
Py_DECREF(obj);
if (c->path.length > 0) {
obj = PyString_FromStringAndSize((char *) c->path.start,
c->path.length);
if (nxt_slow_path(obj == NULL)) {
nxt_alert(task, "Python failed to create string object \"%V\"",
&c->path);
goto fail;
}
pypath = PySys_GetObject((char *) "path");
if (nxt_slow_path(pypath == NULL)) {
nxt_alert(task, "Python failed to get \"sys.path\" list");
goto fail;
}
if (nxt_slow_path(PyList_Insert(pypath, 0, obj) != 0)) {
nxt_alert(task, "Python failed to insert \"%V\" into \"sys.path\"",
&c->path);
goto fail;
}
Py_DECREF(obj);
}
obj = PyCFunction_New(nxt_py_start_resp_method, NULL);
if (nxt_slow_path(obj == NULL)) {
nxt_alert(task,
@@ -366,107 +215,43 @@ nxt_python_start(nxt_task_t *task, nxt_process_data_t *data)
}
nxt_py_environ_ptyp = obj;
obj = Py_BuildValue("[s]", "unit");
if (nxt_slow_path(obj == NULL)) {
nxt_alert(task, "Python failed to create the \"sys.argv\" list");
goto fail;
}
if (nxt_slow_path(PySys_SetObject((char *) "argv", obj) != 0)) {
nxt_alert(task, "Python failed to set the \"sys.argv\" list");
goto fail;
}
Py_CLEAR(obj);
nxt_py_module = nxt_alloca(c->module.length + 1);
nxt_memcpy(nxt_py_module, c->module.start, c->module.length);
nxt_py_module[c->module.length] = '\0';
module = PyImport_ImportModule(nxt_py_module);
if (nxt_slow_path(module == NULL)) {
nxt_alert(task, "Python failed to import module \"%s\"", nxt_py_module);
nxt_python_print_exception();
goto fail;
}
obj = PyDict_GetItemString(PyModule_GetDict(module), "application");
if (nxt_slow_path(obj == NULL)) {
nxt_alert(task, "Python failed to get \"application\" "
"from module \"%s\"", nxt_py_module);
goto fail;
}
if (nxt_slow_path(PyCallable_Check(obj) == 0)) {
nxt_alert(task, "\"application\" in module \"%s\" "
"is not a callable object", nxt_py_module);
goto fail;
}
Py_INCREF(obj);
Py_CLEAR(module);
nxt_py_application = obj;
obj = NULL;
nxt_unit_default_init(task, &python_init);
python_init.callbacks.request_handler = nxt_python_request_handler;
python_init.shm_limit = data->app->shm_limit;
unit_ctx = nxt_unit_init(&python_init);
if (nxt_slow_path(unit_ctx == NULL)) {
goto fail;
}
nxt_python_thread_state = PyEval_SaveThread();
rc = nxt_unit_run(unit_ctx);
nxt_unit_done(unit_ctx);
PyEval_RestoreThread(nxt_python_thread_state);
nxt_python_atexit();
exit(rc);
init->callbacks.request_handler = nxt_python_request_handler;
return NXT_OK;
fail:
Py_XDECREF(obj);
Py_XDECREF(module);
nxt_python_atexit();
return NXT_ERROR;
}
static nxt_int_t
nxt_python_init_strings(void)
int
nxt_python_wsgi_run(nxt_unit_ctx_t *ctx)
{
PyObject *obj;
nxt_uint_t i;
nxt_python_string_t *pstr;
int rc;
for (i = 0; i < nxt_nitems(nxt_python_strings); i++) {
pstr = &nxt_python_strings[i];
nxt_python_thread_state = PyEval_SaveThread();
obj = PyString_FromStringAndSize((char *) pstr->string.start,
pstr->string.length);
if (nxt_slow_path(obj == NULL)) {
return NXT_ERROR;
}
rc = nxt_unit_run(ctx);
PyUnicode_InternInPlace(&obj);
PyEval_RestoreThread(nxt_python_thread_state);
*pstr->object_p = obj;
}
return rc;
}
return NXT_OK;
void
nxt_python_wsgi_done(void)
{
nxt_python_done_strings(nxt_python_strings);
Py_XDECREF(nxt_py_start_resp_obj);
Py_XDECREF(nxt_py_write_obj);
Py_XDECREF(nxt_py_environ_ptyp);
}
@@ -597,29 +382,6 @@ done:
}
static void
nxt_python_atexit(void)
{
nxt_uint_t i;
for (i = 0; i < nxt_nitems(nxt_python_strings); i++) {
Py_XDECREF(*nxt_python_strings[i].object_p);
}
Py_XDECREF(nxt_py_stderr_flush);
Py_XDECREF(nxt_py_application);
Py_XDECREF(nxt_py_start_resp_obj);
Py_XDECREF(nxt_py_write_obj);
Py_XDECREF(nxt_py_environ_ptyp);
Py_Finalize();
if (nxt_py_home != NULL) {
nxt_free(nxt_py_home);
}
}
static PyObject *
nxt_python_create_environ(nxt_task_t *task)
{
@@ -1386,28 +1148,6 @@ nxt_py_input_next(PyObject *self)
}
static void
nxt_python_print_exception(void)
{
PyErr_Print();
#if PY_MAJOR_VERSION == 3
/* The backtrace may be buffered in sys.stderr file object. */
{
PyObject *result;
result = PyObject_CallFunction(nxt_py_stderr_flush, NULL);
if (nxt_slow_path(result == NULL)) {
PyErr_Clear();
return;
}
Py_DECREF(result);
}
#endif
}
static int
nxt_python_write(nxt_python_run_ctx_t *ctx, PyObject *bytes)
{