Build a proper Python package

.github/workflows/tests.yml

@@ -6,8 +6,6 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-      - name: Install Build Dependencies
-        run: sudo apt-get -y install doxygen swig
       - name: Checkout
         uses: actions/checkout@v3
       - name: Build
@@ -19,3 +17,16 @@ jobs:
       - name: Run tests
         run: |
           (cd build && make test)
+  pytest:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Build Dependencies
+        run: sudo apt-get -y install doxygen swig
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install
+        run: |
+          pip install -r requirements.dev.txt
+          pip install .
+      - name: Run tests
+        run: pytest

CMakeLists.txt

@@ -23,6 +23,11 @@ include(CheckLibraryExists)
 include(TestBigEndian)
 include(GNUInstallDirs)
 
+# Statically link the Python package
+if(CALL_FROM_SETUP_PY)
+  set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+endif()
+
 CHECK_INCLUDE_FILE(unistd.h HAVE_UNISTD_H)
 CHECK_INCLUDE_FILE(sys/types.h HAVE_SYS_TYPES_H)
 CHECK_INCLUDE_FILE(sys/stat.h HAVE_SYS_STAT_H)
@@ -75,12 +80,17 @@ configure_file(sphinx_config.h.in include/sphinxbase/sphinx_config.h)
 add_definitions(-DHAVE_CONFIG_H)
 
 add_subdirectory(src)
-add_subdirectory(include)
-add_subdirectory(model)
-add_subdirectory(programs)
-add_subdirectory(doc)
-add_subdirectory(test)
-add_subdirectory(swig)
+# Only build SWIG and Python if we are building the package
+if(CALL_FROM_SETUP_PY)
+  add_subdirectory(swig)
+else()
+  # Don't build or install these in Python
+  add_subdirectory(model)
+  add_subdirectory(doc)
+  add_subdirectory(include)
+  add_subdirectory(programs)
+  add_subdirectory(test)
+  configure_file(pocketsphinx.pc.in pocketsphinx.pc @ONLY)
+  install(FILES ${CMAKE_BINARY_DIR}/pocketsphinx.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
+endif()
 
-configure_file(pocketsphinx.pc.in pocketsphinx.pc @ONLY)
-install(FILES ${CMAKE_BINARY_DIR}/pocketsphinx.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)

README.md

@@ -10,9 +10,9 @@ the art.  I am making a release, because people are nonetheless using
 it, and there are a number of historical errors in the build system
 and API which needed to be corrected.
 
-The version number is strange because it needs to be there was a "release"
-that people are using called 5prealpha,
-and we will use proper [semantic versioning](https://semver.org/) from now on.
+The version number is strangely large because there was a "release"
+that people are using called 5prealpha, and we will use proper
+[semantic versioning](https://semver.org/) from now on.
 
 **Please see the LICENSE file for terms of use.**
 

pyproject.toml

@@ -0,0 +1,8 @@
+[build-system]
+requires = [
+    "wheel",
+    "setuptools>=45",
+    "setuptools_scm[toml]>=6.0",
+    "cmake_build_extension",
+]
+build-backend = "setuptools.build_meta"

requirements.dev.txt

@@ -0,0 +1,2 @@
+cmake-build-extension~=0.5.1
+pytest~=7.1.2

setup.cfg

@@ -0,0 +1,16 @@
+[metadata]
+name = pocketsphinx5
+version = 5.0.0rc0
+description = Official Python bindings for PocketSphinx5
+long_description = file: README.md
+long_description_content_type = text/markdown
+author = David Huggins-Daines
+author_email = [email protected]
+license = MIT
+platforms = any
+url = https://github.com/cmusphinx/pocketsphinx
+project_urls =
+    Source  = https://github.com/cmusphinx/pocketsphinx
+    Tracker = https://github.com/cmusphinx/pocketsphinx/issues
+keywords = 
+classifiers =

setup.py

@@ -0,0 +1,38 @@
+import inspect
+import os
+import sys
+from pathlib import Path
+
+import cmake_build_extension
+import setuptools
+
+# This example is compliant with PEP517 and PEP518. It uses the setup.cfg file to store
+# most of the package metadata. However, build extensions are not supported and must be
+# configured in the setup.py.
+setuptools.setup(
+    ext_modules=[
+        cmake_build_extension.CMakeExtension(
+            # This could be anything you like, it is used to create build folders
+            name="PocketSphinx5",
+            # Name of the resulting package name (import mymath_swig)
+            install_prefix="pocketsphinx5",
+            # Selects the folder where the main CMakeLists.txt is stored
+            # (it could be a subfolder)
+            source_dir=str(Path(__file__).parent.absolute()),
+            cmake_configure_options=[
+                # This option points CMake to the right Python interpreter, and helps
+                # the logic of FindPython3.cmake to find the active version
+                f"-DPython3_ROOT_DIR={Path(sys.prefix)}",
+                "-DCALL_FROM_SETUP_PY:BOOL=ON",
+                "-DBUILD_SHARED_LIBS:BOOL=OFF",
+            ]
+        ),
+    ],
+    cmdclass=dict(
+        # Enable the CMakeExtension entries defined above
+        build_ext=cmake_build_extension.BuildExtension,
+        # If the setup.py or setup.cfg are in a subfolder wrt the main CMakeLists.txt,
+        # you can use the following custom command to create the source distribution.
+        # sdist=cmake_build_extension.GitSdistFolder
+    ),
+)

src/CMakeLists.txt

@@ -108,4 +108,7 @@ set_property(TARGET pocketsphinx PROPERTY
   COMPILE_DEFINITIONS POCKETSPHINX_EXPORTS;SPHINXBASE_EXPORTS
 )
 
-install(TARGETS pocketsphinx LIBRARY)
+# No, do not install this when building Python
+if(NOT CALL_FROM_SETUP_PY)
+  install(TARGETS pocketsphinx LIBRARY)
+endif()

swig/CMakeLists.txt

@@ -1,7 +1,16 @@
-find_package(SWIG)
-
+# Set SWIG policies
+cmake_policy(SET CMP0078 NEW)
+cmake_policy(SET CMP0086 NEW)
+find_package(SWIG 4.0)
 if(SWIG_FOUND)
+  set(UseSWIG_MODULE_VERSION 2)
   # mysteriously necessary
   include(${SWIG_USE_FILE})
   add_subdirectory(python)
 endif()
+
+set_property(
+    SOURCE pocketsphinx.i
+    PROPERTY SWIG_DEPENDS
+    ps_decoder.i ps_lattice.i typemaps.i iterators.i
+    cmd_ln.i fe.i feat.i fsg_model.i jsgf.i ngram_model.i logmath.i)    

swig/pocketsphinx.i

@@ -62,7 +62,6 @@ negative error code."
 
 %include typemaps.i
 %include iterators.i
-%import sphinxbase.i
 
 #if SWIGPYTHON
 %include pybuffer.i
@@ -72,6 +71,8 @@ negative error code."
 typedef cmd_ln_t Config;
 typedef feat_t Feature;
 typedef fe_t FrontEnd;
+typedef jsgf_t Jsgf;
+typedef jsgf_rule_t JsgfRule;
 typedef fsg_model_t FsgModel;
 typedef logmath_t LogMath;
 typedef ngram_model_t NGramModel;
@@ -100,13 +101,47 @@ typedef ngram_model_t NGramModelSet;
 #endif
 
 #include <pocketsphinx.h>
+#include <sphinxbase/cmd_ln.h>
+#include <sphinxbase/err.h>
+#include <sphinxbase/fe.h>
+#include <sphinxbase/feat.h>
+#include <sphinxbase/jsgf.h>
+#include <sphinxbase/ngram_model.h>
 
+typedef cmd_ln_t Config;
+typedef jsgf_t Jsgf;
+typedef jsgf_rule_t JsgfRule;
+typedef feat_t Feature;
+typedef fe_t FrontEnd;
+typedef fsg_model_t FsgModel;
+typedef logmath_t LogMath;
+typedef ngram_model_t NGramModel;
+typedef ngram_model_t NGramModelSet;
 typedef ps_decoder_t Decoder;
 typedef ps_decoder_t SegmentList;
 typedef ps_decoder_t NBestList;
 typedef ps_lattice_t Lattice;
 %}
 
+%nodefaultctor Config;
+
+typedef struct {} Config;
+typedef struct {} FrontEnd;
+typedef struct {} Feature;
+typedef struct {} FsgModel;
+typedef struct {} JsgfRule;
+typedef struct {} NGramModel;
+typedef struct {} LogMath;
+
+sb_iterator(NGramModelSet, ngram_model_set_iter, NGramModel);
+sb_iterator(Jsgf, jsgf_rule_iter, JsgfRule)
+
+sb_iterable(NGramModelSet, NGramModelSet, ngram_model_set_iter, ngram_model_set_iter, NGramModel)
+sb_iterable(Jsgf, Jsgf, jsgf_rule_iter, jsgf_rule_iter, JsgfRule)
+
+typedef struct {} NGramModelSet;
+typedef struct {} Jsgf;
+
 %typemap(cscode) Segment %{
     public override string ToString() {
 	return Word + " " + StartFrame + " " + EndFrame + " " + Prob;
@@ -220,3 +255,10 @@ typedef struct {} SegmentList;
 
 %include ps_decoder.i
 %include ps_lattice.i
+%include cmd_ln.i
+%include fe.i
+%include feat.i
+%include fsg_model.i
+%include jsgf.i
+%include ngram_model.i
+%include logmath.i

swig/python/CMakeLists.txt

@@ -1,15 +1,54 @@
 find_package(Python3 COMPONENTS Interpreter Development)
-set_property(SOURCE ../pocketsphinx.i PROPERTY SWIG_MODULE_NAME pocketsphinx)
-swig_add_library(pocketsphinx_python
+
+# Handle where to install the resulting Python package
+if(CALL_FROM_SETUP_PY)
+  # The CMakeExtension will set CMAKE_INSTALL_PREFIX to the root
+  # of the resulting wheel archive
+  set(POCKETSPHINX_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX})
+else()
+  message(FATAL_ERROR
+    "Please do not try to build the Python module outside of setup.py")
+endif()
+
+# Set the Python module name (.py file) properly
+set_property(SOURCE ../pocketsphinx.i PROPERTY SWIG_MODULE_NAME bindings)
+# SWIG-ify and compile the bindings
+swig_add_library(bindings
+  TYPE MODULE
   LANGUAGE python
+  OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/pocketsphinx5
+  OUTFILE_DIR ${CMAKE_CURRENT_BINARY_DIR}
   SOURCES
   ../pocketsphinx.i
   )
-swig_link_libraries(pocketsphinx_python pocketsphinx Python3::Python)
-set_property(SOURCE ../sphinxbase.i PROPERTY SWIG_MODULE_NAME sphinxbase)
-swig_add_library(sphinxbase_python
-  LANGUAGE python
-  SOURCES
-  ../sphinxbase.i
-  )
-swig_link_libraries(sphinxbase_python pocketsphinx Python3::Python)
+# Link the bindings with pocketsphinx and Python
+target_link_libraries(bindings PRIVATE pocketsphinx Python3::Python)
+# Build the library in the proper place and not the weird random CMake place
+set_target_properties(bindings PROPERTIES
+  LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/pocketsphinx5)
+# This does something, not sure what
+set_property(TARGET bindings PROPERTY SWIG_USE_TARGET_INCLUDE_DIRECTORIES TRUE)
+# Enable parsing the doxygen comments (FIXME: does this work?)
+set_property(TARGET bindings PROPERTY SWIG_COMPILE_OPTIONS -doxygen)
+
+# Installation stuff
+# Get the autogenerated Python file
+get_property(WRAPPER_PY_FILE
+  TARGET bindings
+  PROPERTY SWIG_SUPPORT_FILES)
+# Install the autogenerated Python file
+install(
+  FILES ${WRAPPER_PY_FILE}
+  DESTINATION ${POCKETSPHINX_INSTALL_PREFIX}
+  COMPONENT bindings)
+# Install the __init__ply file
+install(
+  FILES __init__.py
+  DESTINATION ${POCKETSPHINX_INSTALL_PREFIX})
+# Install the SWIG library
+install(
+  TARGETS bindings
+  COMPONENT bindings
+  LIBRARY DESTINATION ${POCKETSPHINX_INSTALL_PREFIX}
+  ARCHIVE DESTINATION ${POCKETSPHINX_INSTALL_PREFIX}
+  RUNTIME DESTINATION ${POCKETSPHINX_INSTALL_PREFIX})

swig/python/__init__.py

@@ -34,4 +34,4 @@
 # ====================================================================
 
 
-from .pocketsphinx import *
+from .bindings import *