Skip to content

Commit

Permalink
Fix build
Browse files Browse the repository at this point in the history 
This commit allows `pyodbc` to build. The following items have been
updated.

- `pyproject.toml` has been modified to include `numpy` as a hard
  _build_ dependency, but as an optional install dependency for a user.
- `setup.py` has been modified to include `numpy` as it is a hard build
  requirement as defined in the `pyproject.toml` file.
- `src/cursor.cpp` has been updated to move the enum to the
  corresponding `src/cursor.h` file.
- `src/cursor.h` also includes the `Cursor_Validate` object.
- `src/npcontainer.{cpp,h}` have been updated to ensure the build
  happens.
  • Loading branch information
@ndmlny-qs
ndmlny-qs committed Sep 13, 2023
1 parent 131a90f commit 2e41543
Show file tree
Hide file tree
Showing 5 changed files with 14 additions and 253 deletions.
5 changes: 4 additions & 1 deletion pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@ classifiers=['Development Status :: 5 - Production/Stable',
[project.urls]
Homepage = "https://github.com/mkleehammer/pyodbc"

[project.optional-dependencies]
numpy = ["numpy"]

[build-system]
requires = ["setuptools"]
requires = ["setuptools", "numpy"]
build-backend = "setuptools.build_meta"
19 changes: 4 additions & 15 deletions setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,10 @@
from pathlib import Path
from inspect import cleandoc

import numpy
from setuptools import setup
from setuptools.extension import Extension

HAS_NUMPY = False


def _getversion():
# CAREFUL: We need the version in this file so we can set it in a C macro to set
Expand All @@ -36,16 +35,7 @@ def _getversion():
def main():
settings = get_compiler_settings()

try:
import numpy
HAS_NUMPY = True
except ImportWarning:
raise ImportWarning("NumPy was not found, compiling pyodbc without NumPy support.")

files = [
relpath(join('src', f)) for f in os.listdir('src') if f.endswith('.cpp')
and (f != 'npcontainer.cpp' or HAS_NUMPY)
]
files = [relpath(join('src', f)) for f in os.listdir('src') if f.endswith('.cpp')]

if exists('MANIFEST'):
os.remove('MANIFEST')
Expand Down Expand Up @@ -93,9 +83,8 @@ def get_compiler_settings():
'include_dirs': [],
'define_macros': [('PYODBC_VERSION', VERSION)]
}
if HAS_NUMPY:
settings['include_dirs'].append(numpy.get_include())
settings['define_macros'].append(('WITH_NUMPY', '1'))
settings['include_dirs'].append(numpy.get_include())
settings['define_macros'].append(('WITH_NUMPY', '1'))

if os.name == 'nt':
settings['extra_compile_args'].extend([
Expand Down
8 changes: 0 additions & 8 deletions src/cursor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,6 @@
#include "npcontainer.h"
#endif

enum
{
CURSOR_REQUIRE_CNXN = 0x00000001,
CURSOR_REQUIRE_OPEN = 0x00000003, // includes _CNXN
CURSOR_REQUIRE_RESULTS = 0x00000007, // includes _OPEN
CURSOR_RAISE_ERROR = 0x00000010,
};

inline bool StatementIsValid(Cursor* cursor)
{
return cursor->cnxn != 0 && ((Connection*)cursor->cnxn)->hdbc != SQL_NULL_HANDLE && cursor->hstmt != SQL_NULL_HANDLE;
Expand Down
230 changes: 5 additions & 225 deletions src/npcontainer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
Py_ssize_t iopro_text_limit = -1;

// -----------------------------------------------------------------------------

bool pyodbc_tracing_enabled = false;

void pyodbc_trace_func(const char* file, int line, const char* fmt, ...)
Expand All @@ -52,6 +53,7 @@ void pyodbc_trace_func(const char* file, int line, const char* fmt, ...)

#define CHECK_ALLOC_GUARDS(...) {}


namespace {
inline size_t
limit_text_size(size_t sz)
Expand Down Expand Up @@ -1360,7 +1362,7 @@ perform_array_query(query_desc& result, Cursor* cur, npy_intp nrows, bool lower,
chunk_size = static_cast<size_t>(nrows);
}

I(cur->hstmt != SQL_NULL_HANDLE && cur->colinfos != 0);
assert(cur->hstmt != SQL_NULL_HANDLE && cur->colinfos != 0);

if (cur->cnxn->hdbc == SQL_NULL_HANDLE)
{
Expand Down Expand Up @@ -1568,197 +1570,6 @@ create_fill_dictarray(Cursor* cursor, npy_intp nrows, const char* null_suffix)
return dictarray;
}


static PyArray_Descr*
query_desc_to_record_dtype(query_desc &qd, const char *null_suffix)
/*
Build a record dtype from the column information in a query
desc.
returns the dtype (PyArray_Descr) on success with a reference
count. 0 if something failed. On failure the appropriate python
exception will be already raised.
In order to create the structured dtype, PyArray_DescrConverter is
called passing a dictionary that maps the fields.
*/
{
PyObject* record_dict = 0;

record_dict = PyDict_New();

if (!record_dict)
return 0;

long offset = 0;
for (std::vector<column_desc>::iterator it = qd.columns_.begin();
it < qd.columns_.end(); ++it) {
PyObject *field_desc = PyTuple_New(2);

if (!field_desc) {
Py_DECREF(record_dict);
return 0; /* out of memory? */
}

// PyTuple_SET_ITEM steals the reference, we want to keep one
// reference for us. We don't want the extra checks made by
// PyTuple_SetItem.
Py_INCREF(it->npy_type_descr_);
PyTuple_SET_ITEM(field_desc, 0, (PyObject*)it->npy_type_descr_);
PyTuple_SET_ITEM(field_desc, 1, PyInt_FromLong(offset));

int not_inserted = PyDict_SetItemString(record_dict, (const char*) it->sql_name_,
field_desc);
Py_DECREF(field_desc);
if (not_inserted) {
Py_DECREF(record_dict);
return 0; /* out of memory? */
}

offset += it->npy_type_descr_->elsize;

// handle nulls...
if (it->npy_array_nulls_) {
field_desc = PyTuple_New(2);
if (!field_desc)
{
Py_DECREF(record_dict);
return 0;
}

PyArray_Descr *descr = PyArray_DESCR(it->npy_array_nulls_);
Py_INCREF(descr);
PyTuple_SET_ITEM(field_desc, 0, (PyObject*)descr);
PyTuple_SET_ITEM(field_desc, 1, PyInt_FromLong(offset));
char null_column_name[350];
snprintf(null_column_name, sizeof(null_column_name),
"%s%s", it->sql_name_, null_suffix);

not_inserted = PyDict_SetItemString(record_dict, null_column_name, field_desc);
Py_DECREF(field_desc);
if (not_inserted) {
Py_DECREF(record_dict);
return 0;
}
offset += descr->elsize;
}
}

PyArray_Descr *dtype=0;
int success = PyArray_DescrConverter(record_dict, &dtype);
Py_DECREF(record_dict);
if (!success) {
RaiseErrorV(0, ProgrammingError,
"Failed conversion from dict type into a NumPy record dtype");
return 0;
}

return dtype;
}

static PyArrayObject*
query_desc_to_sarray(query_desc &qd, const char *null_suffix)
/*
Build a sarray (structured array) from a query_desc.
*/
{
// query_desc contains "column-wise" results as NumPy arrays. In a
// sarray we want the data row-wise (structured layout). This
// means a whole new array will need to be allocated and memory
// copying will be needed.

// 1. build the record dtype.
PyArray_Descr *dtype = query_desc_to_record_dtype(qd, null_suffix);

if (!dtype) {
TRACE_NOLOC("WARN: failed to create record dtype.\n");
return 0;
}

// 2. build the NumPy Array. It is not needed to clear any data
// (even string data) as everything will be overwritten when
// copying the column arrays. The column arrays where already
// properly initialized before fetching the data.
npy_intp dims = (npy_intp)qd.allocated_results_count_;
PyArrayObject* sarray = reinterpret_cast<PyArrayObject*>(PyArray_SimpleNewFromDescr(1, &dims, dtype));
// note: dtype got its reference stolen, but it is still valid as
// long as the array is valid. The reference is stolen even if the
// array fails to create (according to NumPy source code).

if (!sarray) {
TRACE_NOLOC("WARN: failed to create structured array.\n");
return 0;
}

// 3. copy the data into the structured array. Note: the offsets
// will be the same as in the record array by construction.
{
PyNoGIL no_gil;
long offset = 0;
for (std::vector<column_desc>::iterator it = qd.columns_.begin();
it < qd.columns_.end(); ++it)
{
size_t sarray_stride = PyArray_ITEMSIZE(sarray);
char *sarray_data = PyArray_BYTES(sarray) + offset;
size_t carray_stride = PyArray_ITEMSIZE(it->npy_array_);
char *carray_data = PyArray_BYTES(it->npy_array_);
// this approach may not be the most efficient, but it is good enough for now.
// TODO: make the transform in a way that is sequential on the write stream
for (size_t i = 0; i < qd.allocated_results_count_; ++i)
{
memcpy(sarray_data, carray_data, carray_stride);
sarray_data += sarray_stride;
carray_data += carray_stride;
}

offset += carray_stride;

if (it->npy_array_nulls_)
{
// TODO: refactor this code that is duplicated
sarray_stride = PyArray_ITEMSIZE(sarray);
sarray_data = PyArray_BYTES(sarray) + offset;
carray_stride = PyArray_ITEMSIZE(it->npy_array_nulls_);
carray_data = PyArray_BYTES(it->npy_array_nulls_);
// this approach may not be the most efficient, but it is good enough for now.
// TODO: make the transform in a way that is sequential on the write stream
for (size_t i = 0; i < qd.allocated_results_count_; ++i)
{
memcpy(sarray_data, carray_data, carray_stride);
sarray_data += sarray_stride;
carray_data += carray_stride;
}

offset += carray_stride;
}
}
}

return sarray;
}

static PyObject*
create_fill_sarray(Cursor* cursor, npy_intp nrows, const char* null_suffix)
{
int error;
query_desc qd;

error = perform_array_query(qd, cursor, nrows, lowercase(), null_suffix != 0);
if (error) {
TRACE_NOLOC("perform_querydesc returned %d errors\n", error);
return 0;
}

TRACE_NOLOC("\nBuilding sarray\n");
PyObject *sarray = reinterpret_cast<PyObject*>(query_desc_to_sarray(qd, null_suffix));
if (!sarray) {
TRACE_NOLOC("WARN: Failed to build sarray from the query results.\n");
}

return sarray;
}


// -----------------------------------------------------------------------------
// Method implementation
// -----------------------------------------------------------------------------
Expand All @@ -1769,36 +1580,6 @@ static char *Cursor_npfetch_kwnames[] = {
};


//
// The main cursor.fetchsarray() method
//
PyObject*
Cursor_fetchsarray(PyObject *self, PyObject *args, PyObject *kwargs)
{
Cursor* cursor = Cursor_Validate(self, CURSOR_REQUIRE_RESULTS | CURSOR_RAISE_ERROR);
if (!cursor)
return 0;

TRACE("\n\nParse tuple\n");
ssize_t nrows = -1;
const char *null_suffix = "_isnull";
PyObject *return_nulls = NULL;
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|nOs", Cursor_npfetch_kwnames,
&nrows, &return_nulls, &null_suffix))
return 0;

bool preserve_nulls = return_nulls ? PyObject_IsTrue(return_nulls) : false;

TRACE_NOLOC("\n\nCursor fetchsarray\n\tnrows:%d\n\treturn_nulls:%s\n\tnull_suffix:%s\n\thandle:%p\n\tunicode_results:%s\n",
(int)nrows, preserve_nulls?"Yes":"No", null_suffix, (void*)cursor->hstmt,
cursor->cnxn->unicode_results?"Yes":"No");
npy_intp arg = nrows;
PyObject* rv = create_fill_sarray(cursor, arg, preserve_nulls?null_suffix:0);
TRACE_NOLOC("\nCursor fetchsarray done.\n\tsarray: %p\n\n", rv);

return rv;
}

//
// The main cursor.fetchdict() method
//
Expand Down Expand Up @@ -1826,8 +1607,8 @@ Cursor_fetchdictarray(PyObject* self, PyObject* args, PyObject *kwargs)
bool preserve_nulls = return_nulls?PyObject_IsTrue(return_nulls):false;
TRACE("Foo\n");
TRACE_NOLOC("\n\nCursor fetchdictarray\n\tnrows:%d\n\treturn_nulls:%s\n\tnull_suffix:%s\n\thandle:%p\n\tunicode_results:%s\n",
(int)nrows, preserve_nulls?"yes":"no", null_suffix, (void*)cursor->hstmt,
cursor->cnxn->unicode_results?"Yes":"No");
(int)nrows, preserve_nulls?"yes":"no", null_suffix, (void*)cursor->hstmt);
/*cursor->cnxn->unicode_results?"Yes":"No");*/
npy_intp arg = nrows;
PyObject *rv = create_fill_dictarray(cursor, arg, preserve_nulls?null_suffix:0);
TRACE_NOLOC("\nCursor fetchdictarray done.\n\tdictarray: %p\n\n", rv);
Expand Down Expand Up @@ -1878,7 +1659,6 @@ char fetchdictarray_doc[] =
"--------\n" \
"fetchmany : Fetch rows into a Python list of rows.\n" \
"fetchall : Fetch the remaining rows into a Python lis of rows.\n" \
"fetchsarray : Fetch rows into a NumPy structured ndarray.\n" \
"\n";


Expand Down
5 changes: 1 addition & 4 deletions src/npcontainer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,10 @@ int NpContainer_init();
void NpContainer_init();
#endif

PyObject* Cursor_fetchsarray(PyObject* self, PyObject* args, PyObject *kwargs);

PyObject* Cursor_fetchdictarray(PyObject* self, PyObject* args, PyObject *kwargs);
PyObject *Cursor_fetchdictarray(PyObject *self, PyObject *args, PyObject *kwargs);

extern char fetchdictarray_doc[];

extern Py_ssize_t iopro_text_limit;

#endif // _NPCONTAINER_H_

0 comments on commit 2e41543

Please sign in to comment.