Merge pull request #11872 from jefferyto/python3-makefile-cleanup

python3: Makefile cleanup

1 files changed, 130 insertions, 72 deletions

diff --git a/lang/python/python3-package.mk b/lang/python/python3-package.mk
index 97589b563..b16773263 100644
--- a/lang/python/python3-package.mk
+++ b/lang/python/python3-package.mk
@@ -5,12 +5,12 @@
 # See /LICENSE for more information.
 #
 
-# Note: include this after `include $(TOPDIR)/rules.mk in your package Makefile
+# Note: include this file after `include $(TOPDIR)/rules.mk in your package Makefile
+
 python3_mk_path:=$(dir $(lastword $(MAKEFILE_LIST)))
 include $(python3_mk_path)python3-host.mk
 
 PYTHON3_DIR:=$(STAGING_DIR)/usr
-PYTHON3_BIN_DIR:=$(PYTHON3_DIR)/bin
 PYTHON3_INC_DIR:=$(PYTHON3_DIR)/include/python$(PYTHON3_VERSION)
 PYTHON3_LIB_DIR:=$(PYTHON3_DIR)/lib/python$(PYTHON3_VERSION)
 
@@ -27,19 +27,106 @@ CONFIGURE_ARGS += \
 	_python_prefix="/usr" \
 	_python_exec_prefix="/usr"
 
-PKG_USE_MIPS16:=0
-# This is required in addition to PKG_USE_MIPS16:=0 because otherwise MIPS16
-# flags are inherited from the Python base package (via sysconfig module)
-ifdef CONFIG_USE_MIPS16
-  TARGET_CFLAGS += -mno-mips16 -mno-interlink-mips16
-endif
+PYTHON3_VARS = \
+	CC="$(TARGET_CC)" \
+	CCSHARED="$(TARGET_CC) $(FPIC)" \
+	CXX="$(TARGET_CXX)" \
+	LD="$(TARGET_CC)" \
+	LDSHARED="$(TARGET_CC) -shared" \
+	CFLAGS="$(TARGET_CFLAGS)" \
+	CPPFLAGS="$(TARGET_CPPFLAGS) -I$(PYTHON3_INC_DIR)" \
+	LDFLAGS="$(TARGET_LDFLAGS) -lpython$(PYTHON3_VERSION)" \
+	_PYTHON_HOST_PLATFORM=linux2 \
+	__PYVENV_LAUNCHER__="/usr/bin/$(PYTHON3)" \
+	PYTHONPATH="$(PYTHON3PATH)" \
+	PYTHONDONTWRITEBYTECODE=1 \
+	PYTHONOPTIMIZE="" \
+	_python_sysroot="$(STAGING_DIR)" \
+	_python_prefix="/usr" \
+	_python_exec_prefix="/usr"
 
-define Py3Shebang
-$(SED) "1"'!'"b;s,^#"'!'".*python.*,#"'!'"/usr/bin/python3," -i --follow-symlinks $(1)
+# $(1) => directory of python script
+# $(2) => python script and its arguments
+# $(3) => additional variables
+define Python3/Run
+	cd "$(if $(strip $(1)),$(strip $(1)),.)" && \
+	$(PYTHON3_VARS) \
+	$(3) \
+	$(HOST_PYTHON3_BIN) $(2)
 endef
 
-define Py3Package
+# $(1) => build subdir
+# $(2) => additional arguments to setup.py
+# $(3) => additional variables
+define Python3/ModSetup
+	$(INSTALL_DIR) $(PKG_INSTALL_DIR)/$(PYTHON3_PKG_DIR)
+	$(call Python3/Run, \
+		$(PKG_BUILD_DIR)/$(strip $(1)), \
+		setup.py $(2), \
+		$(3))
+endef
+
+define Python3/FixShebang
+	$(SED) "1"'!'"b;s,^#"'!'".*python.*,#"'!'"/usr/bin/python3," -i --follow-symlinks $(1)
+endef
+
+# default max recursion is 10
+PYTHON3_COMPILEALL_MAX_RECURSION_LEVEL:=20
+
+# $(1) => directory of python source files to compile
+#
+# XXX [So that you won't goof as I did]
+# Note: Yes, I tried to use the -O & -OO flags here.
+#       However the generated byte-codes were not portable.
+#       So, we just stuck to un-optimized byte-codes,
+#       which is still way better/faster than running
+#       Python sources all the time.
+#
+# Setting a fixed hash seed value is less secure than using
+# random seed values, but is necessary for reproducible builds
+# (for now).
+#
+# Should revisit this when https://bugs.python.org/issue37596
+# (and other related reproducibility issues) are fixed.
+define Python3/CompileAll
+	$(call Python3/Run,, \
+		-m compileall -r "$(PYTHON3_COMPILEALL_MAX_RECURSION_LEVEL)" -b -d '/' $(1),
+		$(if $(SOURCE_DATE_EPOCH),PYTHONHASHSEED="$(SOURCE_DATE_EPOCH)")
+	)
+endef
+
+# $(1) => target directory
+define Python3/DeleteSourceFiles
+	$(FIND) $(1) -type f -name '*.py' -delete
+endef
+
+# $(1) => target directory
+define Python3/DeleteNonSourceFiles
+	$(FIND) $(1) -not -type d -not -name '*.py' -delete
+endef
+
+# $(1) => target directory
+define Python3/DeleteEmptyDirs
+	$(FIND) $(1) -mindepth 1 -empty -type d -not -path '$(1)/CONTROL' -not -path '$(1)/CONTROL/*' -delete
+endef
 
+
+# Py3Package
+
+define Py3Package/filespec/Default
++|$(PYTHON3_PKG_DIR)
+endef
+
+# $(1) => package name
+# $(2) => src directory
+# $(3) => dest directory
+define Py3Package/ProcessFilespec
+	$(eval $(call shexport,Py3Package/$(1)/filespec))
+	$(SHELL) $(python3_mk_path)python-package-install.sh \
+		"$(2)" "$(3)" "$$$$$(call shvar,Py3Package/$(1)/filespec)"
+endef
+
+define Py3Package
   define Package/$(1)-src
     $(call Package/$(1))
     DEPENDS:=
@@ -52,8 +139,9 @@ define Py3Package
   endef
 
   define Package/$(1)-src/description
-    $(call Package/$(1)/description).
-    (Contains the Python3 sources for this package).
+    $$(call Package/$(1)/description)
+
+    This package contains the Python source files for $(1).
   endef
 
   define Package/$(1)-src/config
@@ -61,73 +149,43 @@ define Py3Package
   endef
 
   # Add default PyPackage filespec none defined
-  ifndef Py3Package/$(1)/filespec
-    define Py3Package/$(1)/filespec
-      +|$(PYTHON3_PKG_DIR)
-    endef
+  ifeq ($(origin Py3Package/$(1)/filespec),undefined)
+    Py3Package/$(1)/filespec=$$(Py3Package/filespec/Default)
   endif
 
   ifndef Py3Package/$(1)/install
     define Py3Package/$(1)/install
-		if [ -d $(PKG_INSTALL_DIR)/usr/bin ]; then \
-			$(INSTALL_DIR) $$(1)/usr/bin ; \
-			$(CP) $(PKG_INSTALL_DIR)/usr/bin/* $$(1)/usr/bin/ ; \
-		fi
+	if [ -d $(PKG_INSTALL_DIR)/usr/bin ]; then \
+		$(INSTALL_DIR) $$(1)/usr/bin ; \
+		$(CP) $(PKG_INSTALL_DIR)/usr/bin/* $$(1)/usr/bin/ ; \
+	fi
     endef
   endif
 
   ifndef Package/$(1)/install
-  $(call shexport,Py3Package/$(1)/filespec)
-
-  define Package/$(1)/install
+    define Package/$(1)/install
 	$$(call Py3Package/$(1)/install,$$(1))
-	$(SHELL) $(python3_mk_path)python-package-install.sh "3" \
-		"$(PKG_INSTALL_DIR)" "$$(1)" \
-		"$(HOST_PYTHON3_BIN)" "$$(2)" \
-		"$$$$$$$$$$(call shvar,Py3Package/$(1)/filespec)" && \
- 	if [ -d "$$(1)/usr/bin" ]; then \
-		$(call Py3Shebang,$$(1)/usr/bin/*) ; \
+	$$(call Py3Package/ProcessFilespec,$(1),$(PKG_INSTALL_DIR),$$(1))
+	$(FIND) $$(1) -name '*.exe' -delete
+	$$(call Python3/CompileAll,$$(1))
+	$$(call Python3/DeleteSourceFiles,$$(1))
+	$$(call Python3/DeleteEmptyDirs,$$(1))
+	if [ -d "$$(1)/usr/bin" ]; then \
+		$$(call Python3/FixShebang,$$(1)/usr/bin/*) ; \
 	fi
- endef
+    endef
 
-  define Package/$(1)-src/install
-	$$(call Package/$(1)/install,$$(1),sources)
-  endef
+    define Package/$(1)-src/install
+	$$(call Py3Package/$(1)/install,$$(1))
+	$$(call Py3Package/ProcessFilespec,$(1),$(PKG_INSTALL_DIR),$$(1))
+	$$(call Python3/DeleteNonSourceFiles,$$(1))
+	$$(call Python3/DeleteEmptyDirs,$$(1))
+    endef
   endif # Package/$(1)/install
 endef
 
-# $(1) => commands to execute before running pythons script
-# $(2) => python script and its arguments
-# $(3) => additional variables
-define Build/Compile/HostPy3RunTarget
-	$(call HostPython3, \
-		$(if $(1),$(1);) \
-		CC="$(TARGET_CC)" \
-		CCSHARED="$(TARGET_CC) $(FPIC)" \
-		CXX="$(TARGET_CXX)" \
-		LD="$(TARGET_CC)" \
-		LDSHARED="$(TARGET_CC) -shared" \
-		CFLAGS="$(TARGET_CFLAGS)" \
-		CPPFLAGS="$(TARGET_CPPFLAGS) -I$(PYTHON3_INC_DIR)" \
-		LDFLAGS="$(TARGET_LDFLAGS) -lpython$(PYTHON3_VERSION)" \
-		_PYTHON_HOST_PLATFORM=linux2 \
-		__PYVENV_LAUNCHER__="/usr/bin/$(PYTHON3)" \
-		$(3) \
-		, \
-		$(2) \
-	)
-endef
 
-# $(1) => build subdir
-# $(2) => additional arguments to setup.py
-# $(3) => additional variables
-define Build/Compile/Py3Mod
-	$(INSTALL_DIR) $(PKG_INSTALL_DIR)/$(PYTHON3_PKG_DIR)
-	$(call Build/Compile/HostPy3RunTarget, \
-		cd $(PKG_BUILD_DIR)/$(strip $(1)), \
-		./setup.py $(2), \
-		$(3))
-endef
+# Py3Build
 
 PYTHON3_PKG_SETUP_DIR ?=
 PYTHON3_PKG_SETUP_GLOBAL_ARGS ?=
@@ -136,9 +194,9 @@ PYTHON3_PKG_SETUP_VARS ?=
 
 define Py3Build/Compile/Default
 	$(if $(HOST_PYTHON3_PACKAGE_BUILD_DEPENDS),
-		$(call Build/Compile/HostPy3PipInstall,$(HOST_PYTHON3_PACKAGE_BUILD_DEPENDS))
+		$(call HostPython3/PipInstall,$(HOST_PYTHON3_PACKAGE_BUILD_DEPENDS))
 	)
-	$(call Build/Compile/Py3Mod, \
+	$(call Python3/ModSetup, \
 		$(PYTHON3_PKG_SETUP_DIR), \
 		$(PYTHON3_PKG_SETUP_GLOBAL_ARGS) \
 		install --prefix="/usr" --root="$(PKG_INSTALL_DIR)" \
@@ -149,8 +207,8 @@ endef
 
 Py3Build/Compile=$(Py3Build/Compile/Default)
 
-ifeq ($(BUILD_VARIANT),python3)
-define Build/Compile
-	$(call Py3Build/Compile)
-endef
-endif # python3
+PYTHON3_PKG_BUILD ?= 1
+
+ifeq ($(strip $(PYTHON3_PKG_BUILD)),1)
+  Build/Compile=$(Py3Build/Compile)
+endif