From ccdc6bc530444283a510ceb6a50878493fe356f2 Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean Date: Wed, 10 Jan 2018 09:29:41 +0200 Subject: python,python3: export mk files outside of python package dirs Since `lang/python` is it's own folder of Python packages (for both Python 2 & 3), and these build rules are needed in a lot of packages [especially Python packages], putting them here makes sense architecturally, to be shared. This also helps get rid of the `include_mk` construct which relies on OpenWrt core to provide, and seems like a broken design idea that has persisted for a while. Reason is: it requires that Python 2/3 be built to provide these mk files for other Python packages, which seems like a bad idea. Long-term, there could be an issue where some other feeds would require these mk files [e.g. telephony] for some Python packages. We'll see how we handle this a bit later. For now we limit this to this feed. Signed-off-by: Alexandru Ardelean --- lang/python/python-package.mk | 145 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 lang/python/python-package.mk (limited to 'lang/python/python-package.mk') diff --git a/lang/python/python-package.mk b/lang/python/python-package.mk new file mode 100644 index 000000000..2bcae919e --- /dev/null +++ b/lang/python/python-package.mk @@ -0,0 +1,145 @@ +# +# Copyright (C) 2006-2016 OpenWrt.org +# +# This is free software, licensed under the GNU General Public License v2. +# See /LICENSE for more information. +# + +# Note: include this after `include $(TOPDIR)/rules.mk in your package Makefile + +python_mk_path:=$(dir $(lastword $(MAKEFILE_LIST))) +include $(python_mk_path)python-host.mk + +PYTHON_DIR:=$(STAGING_DIR)/usr +PYTHON_BIN_DIR:=$(PYTHON_DIR)/bin +PYTHON_INC_DIR:=$(PYTHON_DIR)/include/python$(PYTHON_VERSION) +PYTHON_LIB_DIR:=$(PYTHON_DIR)/lib/python$(PYTHON_VERSION) + +PYTHON_PKG_DIR:=/usr/lib/python$(PYTHON_VERSION)/site-packages + +PYTHON:=python$(PYTHON_VERSION) + +PYTHONPATH:=$(PYTHON_LIB_DIR):$(STAGING_DIR)/$(PYTHON_PKG_DIR):$(PKG_INSTALL_DIR)/$(PYTHON_PKG_DIR) + +# These configure args are needed in detection of path to Python header files +# using autotools. +CONFIGURE_ARGS += \ + _python_sysroot="$(STAGING_DIR)" \ + _python_prefix="/usr" \ + _python_exec_prefix="/usr" + +PKG_USE_MIPS16:=0 +# This is required in addition to PKG_USE_MIPS16:=0 because otherwise MIPS16 +# flags are inherited from the Python base package (via sysconfig module) +ifdef CONFIG_USE_MIPS16 + TARGET_CFLAGS += -mno-mips16 -mno-interlink-mips16 +endif + +define PyPackage + + define Package/$(1)-src + $(call Package/$(1)) + TITLE+= (sources) + DEPENDS:=$$$$(foreach dep,$$$$(filter +python-%,$$$$(DEPENDS)),$$$$(dep)-src) + endef + + define Package/$(1)-src/description + $(call Package/$(1)/description). + (Contains the Python sources for this package). + endef + + # Add default PyPackage filespec none defined + ifndef PyPackage/$(1)/filespec + define PyPackage/$(1)/filespec + +|$(PYTHON_PKG_DIR) + endef + endif + + ifndef PyPackage/$(1)/install + define PyPackage/$(1)/install + if [ -d $(PKG_INSTALL_DIR)/usr/bin ]; then \ + $(INSTALL_DIR) $$(1)/usr/bin ; \ + $(CP) $(PKG_INSTALL_DIR)/usr/bin/* $$(1)/usr/bin/ ; \ + fi + endef + endif + + ifndef Package/$(1)/install + $(call shexport,PyPackage/$(1)/filespec) + + define Package/$(1)/install + $(call PyPackage/$(1)/install,$$(1)) + find $(PKG_INSTALL_DIR) -name "*\.exe" | xargs rm -f + if [ -e files/python-package-install.sh ] ; then \ + $(SHELL) files/python-package-install.sh \ + "$(PKG_INSTALL_DIR)" "$$(1)" \ + "$(HOST_PYTHON_BIN)" "$$(2)" \ + "$$$$$$$$$$(call shvar,PyPackage/$(1)/filespec)" ; \ + elif [ -e $(STAGING_DIR)/mk/python-package-install.sh ] ; then \ + $(SHELL) $(STAGING_DIR)/mk/python-package-install.sh \ + "$(PKG_INSTALL_DIR)" "$$(1)" \ + "$(HOST_PYTHON_BIN)" "$$(2)" \ + "$$$$$$$$$$(call shvar,PyPackage/$(1)/filespec)" ; \ + else \ + echo "No 'python-package-install.sh' script found" ; \ + exit 1 ; \ + fi + endef + + define Package/$(1)-src/install + $$(call Package/$(1)/install,$$(1),sources) + endef + endif # Package/$(1)/install +endef + +# $(1) => commands to execute before running pythons script +# $(2) => python script and its arguments +# $(3) => additional variables +define Build/Compile/HostPyRunTarget + $(call HostPython, \ + $(if $(1),$(1);) \ + CC="$(TARGET_CC)" \ + CCSHARED="$(TARGET_CC) $(FPIC)" \ + CXX="$(TARGET_CXX)" \ + LD="$(TARGET_CC)" \ + LDSHARED="$(TARGET_CC) -shared" \ + CFLAGS="$(TARGET_CFLAGS)" \ + CPPFLAGS="$(TARGET_CPPFLAGS) -I$(PYTHON_INC_DIR)" \ + LDFLAGS="$(TARGET_LDFLAGS) -lpython$(PYTHON_VERSION)" \ + _PYTHON_HOST_PLATFORM=linux2 \ + __PYVENV_LAUNCHER__="/usr/bin/$(PYTHON)" \ + $(3) \ + , \ + $(2) \ + ) +endef + +# $(1) => build subdir +# $(2) => additional arguments to setup.py +# $(3) => additional variables +define Build/Compile/PyMod + $(INSTALL_DIR) $(PKG_INSTALL_DIR)/$(PYTHON_PKG_DIR) + $(call Build/Compile/HostPyRunTarget, \ + cd $(PKG_BUILD_DIR)/$(strip $(1)), \ + ./setup.py $(2), \ + $(3)) + find $(PKG_INSTALL_DIR) -name "*\.exe" | xargs rm -f +endef + +define PyBuild/Compile/Default + $(foreach pkg,$(HOST_PYTHON_PACKAGE_BUILD_DEPENDS), + $(call host_python_pip_install_host,$(pkg)) + ) + $(call Build/Compile/PyMod,, \ + install --prefix="/usr" --root="$(PKG_INSTALL_DIR)" \ + --single-version-externally-managed \ + ) +endef + +PyBuild/Compile=$(PyBuild/Compile/Default) + +ifeq ($(BUILD_VARIANT),python) +define Build/Compile + $(call PyBuild/Compile) +endef +endif # python -- cgit From 57987b699b4a02552f2ff916e797a2e930c63d25 Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean Date: Wed, 10 Jan 2018 09:43:00 +0200 Subject: python,python3: simplify path to install shell-script Now that all files are exported, it makes sense to just reference the script directly. Signed-off-by: Alexandru Ardelean --- lang/python/python-package.mk | 18 ++++-------------- lang/python/python3-package.mk | 18 ++++-------------- 2 files changed, 8 insertions(+), 28 deletions(-) (limited to 'lang/python/python-package.mk') diff --git a/lang/python/python-package.mk b/lang/python/python-package.mk index 2bcae919e..d1486d726 100644 --- a/lang/python/python-package.mk +++ b/lang/python/python-package.mk @@ -70,20 +70,10 @@ define PyPackage define Package/$(1)/install $(call PyPackage/$(1)/install,$$(1)) find $(PKG_INSTALL_DIR) -name "*\.exe" | xargs rm -f - if [ -e files/python-package-install.sh ] ; then \ - $(SHELL) files/python-package-install.sh \ - "$(PKG_INSTALL_DIR)" "$$(1)" \ - "$(HOST_PYTHON_BIN)" "$$(2)" \ - "$$$$$$$$$$(call shvar,PyPackage/$(1)/filespec)" ; \ - elif [ -e $(STAGING_DIR)/mk/python-package-install.sh ] ; then \ - $(SHELL) $(STAGING_DIR)/mk/python-package-install.sh \ - "$(PKG_INSTALL_DIR)" "$$(1)" \ - "$(HOST_PYTHON_BIN)" "$$(2)" \ - "$$$$$$$$$$(call shvar,PyPackage/$(1)/filespec)" ; \ - else \ - echo "No 'python-package-install.sh' script found" ; \ - exit 1 ; \ - fi + $(SHELL) $(python_mk_path)python-package-install.sh \ + "$(PKG_INSTALL_DIR)" "$$(1)" \ + "$(HOST_PYTHON_BIN)" "$$(2)" \ + "$$$$$$$$$$(call shvar,PyPackage/$(1)/filespec)" endef define Package/$(1)-src/install diff --git a/lang/python/python3-package.mk b/lang/python/python3-package.mk index 25d98342a..b7aaa645c 100644 --- a/lang/python/python3-package.mk +++ b/lang/python/python3-package.mk @@ -69,20 +69,10 @@ define Py3Package define Package/$(1)/install $(call Py3Package/$(1)/install,$$(1)) find $(PKG_INSTALL_DIR) -name "*\.exe" | xargs rm -f - if [ -e files/python3-package-install.sh ] ; then \ - $(SHELL) files/python3-package-install.sh \ - "$(PKG_INSTALL_DIR)" "$$(1)" \ - "$(HOST_PYTHON3_BIN)" "$$(2)" \ - "$$$$$$$$$$(call shvar,Py3Package/$(1)/filespec)" ; \ - elif [ -e $(STAGING_DIR)/mk/python3-package-install.sh ] ; then \ - $(SHELL) $(STAGING_DIR)/mk/python3-package-install.sh \ - "$(PKG_INSTALL_DIR)" "$$(1)" \ - "$(HOST_PYTHON3_BIN)" "$$(2)" \ - "$$$$$$$$$$(call shvar,Py3Package/$(1)/filespec)" ; \ - else \ - echo "No 'python3-package-install.sh' script found" ; \ - exit 1 ; \ - fi + $(SHELL) $(python3_mk_path)python3-package-install.sh \ + "$(PKG_INSTALL_DIR)" "$$(1)" \ + "$(HOST_PYTHON3_BIN)" "$$(2)" \ + "$$$$$$$$$$(call shvar,Py3Package/$(1)/filespec)" endef define Package/$(1)-src/install -- cgit From f4c098cc85417fdfc39476d0b8fda6981ac3c265 Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean Date: Wed, 10 Jan 2018 09:52:18 +0200 Subject: python,python3: merge package install scripts The only difference just a parameter for Python3 [ -b to compile bytecodes in legacy mode ]. No need to keep 2 almost identical files now that they're exported. I'm a bit scared of that param, since it may get removed at some point. But let's see until then. Signed-off-by: Alexandru Ardelean --- lang/python/python-package-install.sh | 16 ++++--- lang/python/python-package.mk | 2 +- lang/python/python3-package-install.sh | 82 ---------------------------------- lang/python/python3-package.mk | 2 +- 4 files changed, 12 insertions(+), 90 deletions(-) delete mode 100644 lang/python/python3-package-install.sh (limited to 'lang/python/python-package.mk') diff --git a/lang/python/python-package-install.sh b/lang/python/python-package-install.sh index 60c1e273c..d5a50322f 100644 --- a/lang/python/python-package-install.sh +++ b/lang/python/python-package-install.sh @@ -35,11 +35,12 @@ process_filespec() { ) } -src_dir="$1" -dst_dir="$2" -python="$3" -mode="$4" -filespec="$5" +ver="$1" +src_dir="$2" +dst_dir="$3" +python="$4" +mode="$5" +filespec="$6" process_filespec "$src_dir" "$dst_dir" "$filespec" || { echo "process filespec error-ed" @@ -58,13 +59,16 @@ if [ "$mode" == "sources" ] ; then exit 0 fi +legacy= +[ "$ver" == "3" ] && legacy="-b" + # XXX [So that you won't goof as I did] # Note: Yes, I tried to use the -O & -OO flags here. # However the generated byte-codes were not portable. # So, we just stuck to un-optimized byte-codes, # which is still way better/faster than running # Python sources all the time. -$python -m compileall -d '/' $dst_dir || { +$python -m compileall $legacy -d '/' $dst_dir || { echo "python -m compileall err-ed" exit 1 } diff --git a/lang/python/python-package.mk b/lang/python/python-package.mk index d1486d726..8bfd5e37f 100644 --- a/lang/python/python-package.mk +++ b/lang/python/python-package.mk @@ -70,7 +70,7 @@ define PyPackage define Package/$(1)/install $(call PyPackage/$(1)/install,$$(1)) find $(PKG_INSTALL_DIR) -name "*\.exe" | xargs rm -f - $(SHELL) $(python_mk_path)python-package-install.sh \ + $(SHELL) $(python_mk_path)python-package-install.sh "2" \ "$(PKG_INSTALL_DIR)" "$$(1)" \ "$(HOST_PYTHON_BIN)" "$$(2)" \ "$$$$$$$$$$(call shvar,PyPackage/$(1)/filespec)" diff --git a/lang/python/python3-package-install.sh b/lang/python/python3-package-install.sh deleted file mode 100644 index 373f315a1..000000000 --- a/lang/python/python3-package-install.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/sh -set -e - -process_filespec() { - local src_dir="$1" - local dst_dir="$2" - local filespec="$3" - echo "$filespec" | ( - IFS='|' - while read fop fspec fperm; do - local fop=`echo "$fop" | tr -d ' \t\n'` - if [ "$fop" = "+" ]; then - if [ ! -e "${src_dir}${fspec}" ]; then - echo "File not found '${src_dir}${fspec}'" - exit 1 - fi - dpath=`dirname "$fspec"` - if [ -z "$fperm" ]; then - dperm=`stat -c "%a" ${src_dir}${dpath}` - fi - mkdir -p -m$dperm ${dst_dir}${dpath} - echo "copying: '$fspec'" - cp -fpR ${src_dir}${fspec} ${dst_dir}${dpath}/ - if [ -n "$fperm" ]; then - chmod -R $fperm ${dst_dir}${fspec} - fi - elif [ "$fop" = "-" ]; then - echo "removing: '$fspec'" - rm -fR ${dst_dir}${fspec} - elif [ "$fop" = "=" ]; then - echo "setting permissions: '$fperm' on '$fspec'" - chmod -R $fperm ${dst_dir}${fspec} - fi - done - ) -} - -src_dir="$1" -dst_dir="$2" -python="$3" -mode="$4" -filespec="$5" - -process_filespec "$src_dir" "$dst_dir" "$filespec" || { - echo "process filespec error-ed" - exit 1 -} - -if [ "$mode" == "sources" ] ; then - # Copy only python source files - find $dst_dir -not -type d -not -name "*\.py" | xargs rm -f - - # Delete empty folders (if the case) - if [ -d "$dst_dir/usr" ] ; then - find $dst_dir/usr -type d | xargs rmdir --ignore-fail-on-non-empty - rmdir --ignore-fail-on-non-empty $dst_dir/usr - fi - exit 0 -fi - -# XXX [So that you won't goof as I did] -# Note: Yes, I tried to use the -O & -OO flags here. -# However the generated byte-codes were not portable. -# So, we just stuck to un-optimized byte-codes, -# which is still way better/faster than running -# Python sources all the time. -$python -m compileall -b -d '/' $dst_dir || { - echo "python -m compileall err-ed" - exit 1 -} - -# Delete source files and pyc [ un-optimized bytecode files ] -# We may want to make this optimization thing configurable later, but not sure atm -find $dst_dir -type f -name "*\.py" | xargs rm -f - -# Delete empty folders (if the case) -if [ -d "$dst_dir/usr" ] ; then - find $dst_dir/usr -type d | xargs rmdir --ignore-fail-on-non-empty - rmdir --ignore-fail-on-non-empty $dst_dir/usr -fi - -exit 0 diff --git a/lang/python/python3-package.mk b/lang/python/python3-package.mk index b7aaa645c..c5bfd71de 100644 --- a/lang/python/python3-package.mk +++ b/lang/python/python3-package.mk @@ -69,7 +69,7 @@ define Py3Package define Package/$(1)/install $(call Py3Package/$(1)/install,$$(1)) find $(PKG_INSTALL_DIR) -name "*\.exe" | xargs rm -f - $(SHELL) $(python3_mk_path)python3-package-install.sh \ + $(SHELL) $(python3_mk_path)python-package-install.sh "3" \ "$(PKG_INSTALL_DIR)" "$$(1)" \ "$(HOST_PYTHON3_BIN)" "$$(2)" \ "$$$$$$$$$$(call shvar,Py3Package/$(1)/filespec)" -- cgit