Creating Python bindings for C / C ++ libraries using SIP. Part 1

Making a binding over a library in C

Writing a library in C

int foo(char*);

#ifndef FOO_LIB
#define FOO_LIB

int foo(char* str);

#endif

#include <string.h>

#include "foo.h"

int foo(char* str) {
	return strlen(str) * 2;
}

#include <stdio.h>

#include "foo.h"

int main(int argc, char* argv[]) {
	char* str = "0123456789";
	printf("%d\n", foo(str));
}

CC=gcc
CFLAGS=-c
DIR_OUT=bin

all: main

main: main.o libfoo.a
	$(CC) $(DIR_OUT)/main.o -L$(DIR_OUT) -lfoo -o $(DIR_OUT)/main

main.o: makedir main.c
	$(CC) $(CFLAGS) main.c -o $(DIR_OUT)/main.o

libfoo.a: makedir foo.c
	$(CC) $(CFLAGS) foo.c -o $(DIR_OUT)/foo.o
	ar rcs $(DIR_OUT)/libfoo.a $(DIR_OUT)/foo.o

makedir:
	mkdir -p $(DIR_OUT)

clean:
	rm -rf $(DIR_OUT)/*

foo_c_01 /
└── foo
    ├── foo.c
    ├── foo.h
    ├── main.c
    └── Makefile

make

mkdir -p bin
gcc -c main.c -o bin/main.o
gcc -c foo.c -o bin/foo.o
ar rcs bin/libfoo.a bin/foo.o
gcc bin/main.o -Lbin -lfoo -o bin/main

foo_c_01 /
└── foo
    ├── bin
    │ ├── foo.o
    │ ├── libfoo.a
    │ ├── main
    │ └── main.o
    ├── foo.c
    ├── foo.h
    ├── main.c
    └── Makefile

SIP Basics

pip install --user sip

pyproject.toml

foo_c_01 /
├── foo
│ ├── bin
│ │ ├── foo.o
│ │ ├── libfoo.a
│ │ ├── main
│ │ └── main.o
│ ├── foo.c
│ ├── foo.h
│ ├── main.c
│ └── Makefile
└── pyproject.toml

[build-system]
requires = ["sip >=5, <6"]
build-backend = "sipbuild.api"

[tool.sip.metadata]
name = "pyfoo"
version = "0.1"
license = "MIT"

[tool.sip.bindings.pyfoo]
headers = ["foo.h"]
libraries = ["foo"]
include-dirs = ["foo"]
library-dirs = ["foo/bin"]

• requires - a list of packages needed to build our package. The package dependency description format is described in PEP 508 Dependency specification for Python Software Packages . In this case, we only need the sip package version 5.x.
• build-backend , . , Python-, . , , SIP, «sipbuild.api».

• headers - a list of header files that are needed to use the foo library.
• libraries - a list of object files compiled for static linking.
• include-dirs is the path where to look for additional header files besides those that are attached to the C compiler. In this case, where to look for the foo.h file .
• library-dirs is the path where to look for additional object files besides those that are attached to the C compiler. In this case, this is the folder in which the compiled library file foo is created .

pyfoo.sip

foo_c_01 /
├── foo
│ ├── bin
│ │ ├── foo.o
│ │ ├── libfoo.a
│ │ ├── main
│ │ └── main.o
│ ├── foo.c
│ ├── foo.h
│ ├── main.c
│ └── Makefile
├── pyfoo.sip
└── pyproject.toml

%Module(name=foo, language="C")

int foo(char*);

from foo import foo

%Module(name=foo.bar, language="C")
...

from foo.bar import foo

int foo(char*);

We collect and check

$ make

mkdir -p bin
gcc -c main.c -o bin/main.o
gcc -c foo.c -o bin/foo.o
ar rcs bin/libfoo.a bin/foo.o
gcc bin/main.o -Lbin -lfoo -o bin/main

• sip-build . Creates a Python extension object file.
• sip-install . Creates a Python extension object file and installs it.
• sip-sdist. .tar.gz, pip.
• sip-wheel. wheel ( .whl).
• sip-module. , , SIP. , , . , standalone project, , , , .
• sip-distinfo. .dist-info, wheel.

pyfoo_c_01
├── build
│ └── foo
│ ├── apiversions.c
│ ├── array.c
│ ├── array.h
│ ├── bool.cpp
│ ├── build
│ │ └── temp.linux-x86_64-3.8
│ │ ├── apiversions.o
│ │ ├── array.o
│ │ ├── bool.o
│ │ ├── descriptors.o
│ │ ├── int_convertors.o
│ │ ├── objmap.o
│ │ ├── qtlib.o
│ │ ├── sipfoocmodule.o
│ │ ├── siplib.o
│ │ ├── threads.o
│ │ └── voidptr.o
│ ├── descriptors.c
│ ├── foo.cpython-38-x86_64-linux-gnu.so
│ ├── int_convertors.c
│ ├── objmap.c
│ ├── qtlib.c
│ ├── sipAPIfoo.h
│ ├── sipfoocmodule.c
│ ├── sip.h
│ ├── sipint.h
│ ├── siplib.c
│ ├── threads.c
│ └── voidptr.c
├── foo
│ ├── bin
│ │ ├── foo.o
│ │ ├── libfoo.a
│ │ ├── main
│ │ └── main.o
│ ├── foo.c
│ ├── foo.h
│ ├── main.c
│ └── Makefile
├── pyfoo.sip
└── pyproject.toml

/*
 * Module code.
 *
 * Generated by SIP 5.1.1
 */

#include "sipAPIfoo.h"

/* Define the strings used by this module. */
const char sipStrings_foo[] = {
    'f', 'o', 'o', 0,
};

PyDoc_STRVAR(doc_foo, "foo(str) -> int");

static PyObject *func_foo(PyObject *sipSelf,PyObject *sipArgs)
{
    PyObject *sipParseErr = SIP_NULLPTR;

    {
        char* a0;

        if (sipParseArgs(&sipParseErr, sipArgs, "s", &a0))
        {
            int sipRes;

            sipRes = foo(a0);

            return PyLong_FromLong(sipRes);
        }
    }

    /* Raise an exception if the arguments couldn't be parsed. */
    sipNoFunction(sipParseErr, sipName_foo, doc_foo);

    return SIP_NULLPTR;
}

/* This defines this module. */
sipExportedModuleDef sipModuleAPI_foo = {
    0,
    SIP_ABI_MINOR_VERSION,
    sipNameNr_foo,
    0,
    sipStrings_foo,
    SIP_NULLPTR,
    SIP_NULLPTR,
    0,
    SIP_NULLPTR,
    SIP_NULLPTR,
    0,
    SIP_NULLPTR,
    0,
    SIP_NULLPTR,
    SIP_NULLPTR,
    SIP_NULLPTR,
    {SIP_NULLPTR, SIP_NULLPTR, SIP_NULLPTR, SIP_NULLPTR, SIP_NULLPTR,
            SIP_NULLPTR, SIP_NULLPTR, SIP_NULLPTR, SIP_NULLPTR, SIP_NULLPTR},
    SIP_NULLPTR,
    SIP_NULLPTR,
    SIP_NULLPTR,
    SIP_NULLPTR,
    SIP_NULLPTR,
    SIP_NULLPTR,
    SIP_NULLPTR,
    SIP_NULLPTR
};

/* The SIP API and the APIs of any imported modules. */
const sipAPIDef *sipAPI_foo;

/* The Python module initialisation function. */
#if defined(SIP_STATIC_MODULE)
PyObject *PyInit_foo(void)
#else
PyMODINIT_FUNC PyInit_foo(void)
#endif
{
    static PyMethodDef sip_methods[] = {
        {sipName_foo, func_foo, METH_VARARGS, doc_foo},
        {SIP_NULLPTR, SIP_NULLPTR, 0, SIP_NULLPTR}
    };

    static PyModuleDef sip_module_def = {
        PyModuleDef_HEAD_INIT,
        "foo",
        SIP_NULLPTR,
        -1,
        sip_methods,
        SIP_NULLPTR,
        SIP_NULLPTR,
        SIP_NULLPTR,
        SIP_NULLPTR
    };

    PyObject *sipModule, *sipModuleDict;
    /* Initialise the module and get it's dictionary. */
    if ((sipModule = PyModule_Create(&sip_module_def)) == SIP_NULLPTR)
        return SIP_NULLPTR;

    sipModuleDict = PyModule_GetDict(sipModule);

    if ((sipAPI_foo = sip_init_library(sipModuleDict)) == SIP_NULLPTR)
        return SIP_NULLPTR;

    /* Export the module and publish it's API. */
    if (sipExportModule(&sipModuleAPI_foo, SIP_ABI_MAJOR_VERSION, SIP_ABI_MINOR_VERSION, 0) < 0)
    {
        Py_DECREF(sipModule);
        return SIP_NULLPTR;
    }
    /* Initialise the module now all its dependencies have been set up. */
    if (sipInitModule(&sipModuleAPI_foo,sipModuleDict) < 0)
    {
        Py_DECREF(sipModule);
        return SIP_NULLPTR;
    }

    return sipModule;
}

$ sip-sdist

The sdist has been built.

pip install --user pyfoo-0.1.tar.gz

Processing ./pyfoo-0.1.tar.gz
  Installing build dependencies ... done
  Getting requirements to build wheel ... done
    Preparing wheel metadata ... done
Building wheels for collected packages: pyfoo
  Building wheel for pyfoo (PEP 517) ... done
  Created wheel for pyfoo: filename=pyfoo-0.1-cp38-cp38-manylinux1_x86_64.whl size=337289 sha256=762fc578...
  Stored in directory: /home/jenyay/.cache/pip/wheels/54/dc/d8/cc534fff...
Successfully built pyfoo
Installing collected packages: pyfoo
  Attempting uninstall: pyfoo
    Found existing installation: pyfoo 0.1
    Uninstalling pyfoo-0.1:
      Successfully uninstalled pyfoo-0.1
Successfully installed pyfoo-0.1

>>> from foo import foo
>>> foo(b'123456')
12

>>> from foo import foo
>>> foo('123456')

Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
TypeError: foo(str): argument 1 has unexpected type 'str'

sip-wheel

pip install --user --upgrade pyfoo-0.1-cp38-cp38-manylinux1_x86_64.whl

Add conversion rules to char *

%Module(name=foo, language="C")

int foo(char* /Encoding="UTF-8"/);

pip install --user --upgrade pyfoo-0.2-cp38-cp38-manylinux1_x86_64.whl

>>> from foo import foo

>>> foo(b'qwerty')
12

>>> foo('qwerty')
12

>>> foo('')
24

%Module(name=foo, language="C")
%DefaultEncoding "UTF-8"

int foo(char*);

Briefly about project.py. Automate assembly

• build . Called during the call to the sip-build command .
• build_sdist . Called when the sip-sdist command is called .
• build_wheel . Called when the sip-wheel command is called .
• install . Called when the sip-install command is invoked .

import os
import subprocess

from sipbuild import Project

class FooProject(Project):
    def _build_foo(self):
        cwd = os.path.abspath('foo')
        subprocess.run(['make'], cwd=cwd, capture_output=True, check=True)

    def build(self):
        self._build_foo()
        super().build()

    def build_sdist(self, sdist_directory):
        self._build_foo()
        return super().build_sdist(sdist_directory)

    def build_wheel(self, wheel_directory):
        self._build_foo()
        return super().build_wheel(wheel_directory)

    def install(self):
        self._build_foo()
        super().install()

pip install --user --upgrade pyfoo-0.4-cp38-cp38-manylinux1_x86_64.whl

Add command line options to build

import os

from sipbuild import Option, Project

class FooProject(Project):
    """        
       foo.
    """

    def get_options(self):
        """     . """

        tools = ['build', 'install', 'sdist', 'wheel']

        #   .
        options = super().get_options()

        #   
        inc_dir_option = Option('foo_include_dir',
                                help="the directory containing foo.h",
                                metavar="DIR",
                                default=os.path.abspath('foo'),
                                tools=tools)
        options.append(inc_dir_option)

        lib_dir_option = Option('foo_library_dir',
                                help="the directory containing the foo library",
                                metavar="DIR",
                                default=os.path.abspath('foo/bin'),
                                tools=tools)

        options.append(lib_dir_option)

        return options

    def apply_user_defaults(self, tool):
        """    . """

        #     
        super().apply_user_defaults(tool)

        #  ,         
        self.foo_include_dir = os.path.abspath(self.foo_include_dir)
        self.foo_library_dir = os.path.abspath(self.foo_library_dir)

    def update(self, tool):
        """   . """

        #   pyfoo
        # (  pyproject.toml  [tool.sip.bindings.pyfoo])
        foo_bindings = self.bindings['pyfoo']

        #   include_dirs  
        if self.foo_include_dir is not None:
            foo_bindings.include_dirs = [self.foo_include_dir]

        #   library_dirs  
        if self.foo_library_dir is not None:
            foo_bindings.library_dirs = [self.foo_library_dir]

        super().update(tool)

• help , , sip-wheel -h
• metavar — , , . metavar «DIR», , — .
• default — . , , foo , ( ).
• tools — , . sip-build, sip-install, sip-sdist sip-wheel, tools = ['build', 'install', 'sdist', 'wheel'].

[build-system]
requires = ["sip >=5, <6"]
build-backend = "sipbuild.api"

[tool.sip.metadata]
name = "pyfoo"
version = "0.3"
license = "MIT"

[tool.sip.bindings.pyfoo]
headers = ["foo.h"]
libraries = ["foo"]

usr/bin/ld:   -lfoo
collect2: :  ld     1
sip-wheel: Unable to compile the 'foo' module: command 'g++' failed with exit status 1

sip-wheel --foo-include-dir ".../foo" --foo-library-dir ".../foo/bin"

Check the order of calling methods from project.py

from sipbuild import Project

class FooProject(Project):
    def get_options(self):
        print('get_options()')
        options = super().get_options()
        return options

    def apply_user_defaults(self, tool):
        print('apply_user_defaults()')
        super().apply_user_defaults(tool)

    def apply_nonuser_defaults(self, tool):
        print('apply_nonuser_defaults()')
        super().apply_nonuser_defaults(tool)

    def update(self, tool):
        print('update()')
        super().update(tool)

    def build(self):
        print('build()')
        super().build()

    def build_sdist(self, sdist_directory):
        print('build_sdist()')
        return super().build_sdist(sdist_directory)

    def build_wheel(self, wheel_directory):
        print('build_wheel()')
        return super().build_wheel(wheel_directory)

    def install(self):
        print('install()')
        super().install()

[build-system]
requires = ["sip >=5, <6"]
build-backend = "sipbuild.api"

[tool.sip.metadata]
name = "pyfoo"
version = "0.4"
license = "MIT"

[tool.sip.bindings.pyfoo]
headers = ["foo.h"]
libraries = ["foo"]
include-dirs = ["foo"]
library-dirs = ["foo/bin"]

Conclusion to the first part

References

• Examples for this article
• SIP Homepage
• Documentation for SIP