python-setuptools: Add reproducibility patches from Debian

This adds the ability to patch setuptools (and pip), and adds 3
reproducibility patches from Debian[1].
(003-PKG-INFO-output-reproducible.patch addresses the issue identified
in #9039.)

The patching is not perfect, in that the patches are applied to
setuptools and pip after they have been installed, since they are
installed from wheels which are already "precompiled".

Also, patching for the host install cannot be updated in place, for
example if a patch is added or removed.

[1]: https://sources.debian.org/patches/python-setuptools/40.8.0-1/

Signed-off-by: Jeffery To <jeffery.to@gmail.com>

10 files changed, 114 insertions, 2 deletions

diff --git a/lang/python/python-version.mk b/lang/python/python-version.mk
index 418217623..84b8e4f05 100644
--- a/lang/python/python-version.mk
+++ b/lang/python/python-version.mk
@@ -8,7 +8,7 @@
 PYTHON_VERSION:=2.7
 PYTHON_VERSION_MICRO:=16
 
-PYTHON_SETUPTOOLS_PKG_RELEASE:=2
+PYTHON_SETUPTOOLS_PKG_RELEASE:=3
 PYTHON_PIP_PKG_RELEASE:=2
 
 PYTHON_SETUPTOOLS_VERSION:=40.6.2
diff --git a/lang/python/python/Makefile b/lang/python/python/Makefile
index df825bc0d..979ccc771 100644
--- a/lang/python/python/Makefile
+++ b/lang/python/python/Makefile
@@ -174,6 +174,7 @@ define Build/Compile/python-setuptools
 		--ignore-installed \
 		--root=$(PKG_BUILD_DIR)/install-setuptools --prefix=. \
 		$(PKG_BUILD_DIR)/Lib/ensurepip/_bundled/setuptools-$(PYTHON_SETUPTOOLS_VERSION)-py2.py3-none-any.whl
+	$(call PatchDir,$(PKG_BUILD_DIR)/install-setuptools/lib/python$(PYTHON_VERSION)/site-packages,./patches-setuptools,)
 endef
 endif # CONFIG_PACKAGE_python-setuptools
 
@@ -183,6 +184,7 @@ define Build/Compile/python-pip
 		--ignore-installed \
 		--root=$(PKG_BUILD_DIR)/install-pip --prefix=. \
 		$(PKG_BUILD_DIR)/Lib/ensurepip/_bundled/pip-$(PYTHON_PIP_VERSION)-py2.py3-none-any.whl
+	$(call PatchDir,$(PKG_BUILD_DIR)/install-pip/lib/python$(PYTHON_VERSION)/site-packages,./patches-pip,)
 endef
 endif # CONFIG_PACKAGE_python-pip
 
@@ -304,6 +306,14 @@ define Host/Install
 	$(MAKE) -C $(HOST_BUILD_DIR) install
 	$(INSTALL_DIR) $(HOST_PYTHON_DIR)/bin/
 	$(INSTALL_BIN) $(HOST_BUILD_DIR)/Parser/pgen $(HOST_PYTHON_DIR)/bin/pgen2
+  ifeq ($(wildcard $(HOST_PYTHON_PKG_DIR)/.setuptools-patched),)
+	$(call HostPatchDir,$(HOST_PYTHON_PKG_DIR),./patches-setuptools,)
+	touch $(HOST_PYTHON_PKG_DIR)/.setuptools-patched
+  endif
+  ifeq ($(wildcard $(HOST_PYTHON_PKG_DIR)/.pip-patched),)
+	$(call HostPatchDir,$(HOST_PYTHON_PKG_DIR),./patches-pip,)
+	touch $(HOST_PYTHON_PKG_DIR)/.pip-patched
+  endif
 endef
 
 $(eval $(call HostBuild))
diff --git a/lang/python/python/patches-setuptools/001-reproducible.patch b/lang/python/python/patches-setuptools/001-reproducible.patch
new file mode 100644
index 000000000..32edc568d
--- /dev/null
+++ b/lang/python/python/patches-setuptools/001-reproducible.patch
@@ -0,0 +1,16 @@
+https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=848136
+https://sources.debian.org/patches/python-setuptools/40.8.0-1/reproducible.diff/
+
+Index: b/setuptools/command/easy_install.py
+===================================================================
+--- a/setuptools/command/easy_install.py
++++ b/setuptools/command/easy_install.py
+@@ -436,7 +436,7 @@ consider to install to another location,
+             for spec in self.args:
+                 self.easy_install(spec, not self.no_deps)
+             if self.record:
+-                outputs = self.outputs
++                outputs = list(sorted(self.outputs))
+                 if self.root:  # strip any package prefix
+                     root_len = len(self.root)
+                     for counter in range(len(outputs)):
diff --git a/lang/python/python/patches-setuptools/002-sorted-requires.patch b/lang/python/python/patches-setuptools/002-sorted-requires.patch
new file mode 100644
index 000000000..2ad4795b3
--- /dev/null
+++ b/lang/python/python/patches-setuptools/002-sorted-requires.patch
@@ -0,0 +1,16 @@
+https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=804249
+https://sources.debian.org/patches/python-setuptools/40.8.0-1/sorted-requires.diff/
+
+Index: b/setuptools/command/egg_info.py
+===================================================================
+--- a/setuptools/command/egg_info.py
++++ b/setuptools/command/egg_info.py
+@@ -621,7 +621,7 @@ def warn_depends_obsolete(cmd, basename,
+ def _write_requirements(stream, reqs):
+     lines = yield_lines(reqs or ())
+     append_cr = lambda line: line + '\n'
+-    lines = map(append_cr, lines)
++    lines = map(append_cr, sorted(lines))
+     stream.writelines(lines)
+ 
+ 
diff --git a/lang/python/python/patches-setuptools/003-PKG-INFO-output-reproducible.patch b/lang/python/python/patches-setuptools/003-PKG-INFO-output-reproducible.patch
new file mode 100644
index 000000000..15f34dcdb
--- /dev/null
+++ b/lang/python/python/patches-setuptools/003-PKG-INFO-output-reproducible.patch
@@ -0,0 +1,14 @@
+https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=894215
+https://sources.debian.org/patches/python-setuptools/40.8.0-1/PKG-INFO-output-reproducible.diff/
+
+--- a/setuptools/dist.py
++++ b/setuptools/dist.py
+@@ -191,7 +191,7 @@ def write_pkg_file(self, file):
+             self.long_description_content_type
+         )
+     if self.provides_extras:
+-        for extra in self.provides_extras:
++        for extra in sorted(self.provides_extras):
+             write_field('Provides-Extra', extra)
+ 
+ 
diff --git a/lang/python/python3-version.mk b/lang/python/python3-version.mk
index 9eebeb260..784ec1778 100644
--- a/lang/python/python3-version.mk
+++ b/lang/python/python3-version.mk
@@ -12,7 +12,7 @@ PYTHON3_VERSION_MICRO:=2
 
 PYTHON3_VERSION:=$(PYTHON3_VERSION_MAJOR).$(PYTHON3_VERSION_MINOR)
 
-PYTHON3_SETUPTOOLS_PKG_RELEASE:=2
+PYTHON3_SETUPTOOLS_PKG_RELEASE:=3
 PYTHON3_PIP_PKG_RELEASE:=2
 
 PYTHON3_SETUPTOOLS_VERSION:=40.6.2
diff --git a/lang/python/python3/Makefile b/lang/python/python3/Makefile
index 4053ff0c9..d151459ed 100644
--- a/lang/python/python3/Makefile
+++ b/lang/python/python3/Makefile
@@ -178,6 +178,7 @@ define Build/Compile/python3-setuptools
 		--ignore-installed \
 		--root=$(PKG_BUILD_DIR)/install-setuptools --prefix=. \
 		$(PKG_BUILD_DIR)/Lib/ensurepip/_bundled/setuptools-$(PYTHON3_SETUPTOOLS_VERSION)-py2.py3-none-any.whl
+	$(call PatchDir,$(PKG_BUILD_DIR)/install-setuptools/lib/python$(PYTHON3_VERSION)/site-packages,./patches-setuptools,)
 endef
 endif # CONFIG_PACKAGE_python3-setuptools
 
@@ -187,6 +188,7 @@ define Build/Compile/python3-pip
 		--ignore-installed \
 		--root=$(PKG_BUILD_DIR)/install-pip --prefix=. \
 		$(PKG_BUILD_DIR)/Lib/ensurepip/_bundled/pip-$(PYTHON3_PIP_VERSION)-py2.py3-none-any.whl
+	$(call PatchDir,$(PKG_BUILD_DIR)/install-pip/lib/python$(PYTHON3_VERSION)/site-packages,./patches-pip,)
 endef
 endif # CONFIG_PACKAGE_python3-pip
 
@@ -299,6 +301,14 @@ define Host/Install
 	$(MAKE) -C $(HOST_BUILD_DIR) install
 	$(INSTALL_DIR) $(HOST_PYTHON3_DIR)/bin/
 	$(INSTALL_BIN) $(HOST_BUILD_DIR)/Parser/pgen $(HOST_PYTHON3_DIR)/bin/pgen3
+  ifeq ($(wildcard $(HOST_PYTHON3_PKG_DIR)/.setuptools-patched),)
+	$(call HostPatchDir,$(HOST_PYTHON3_PKG_DIR),./patches-setuptools,)
+	touch $(HOST_PYTHON3_PKG_DIR)/.setuptools-patched
+  endif
+  ifeq ($(wildcard $(HOST_PYTHON3_PKG_DIR)/.pip-patched),)
+	$(call HostPatchDir,$(HOST_PYTHON3_PKG_DIR),./patches-pip,)
+	touch $(HOST_PYTHON3_PKG_DIR)/.pip-patched
+  endif
 endef
 
 $(eval $(call HostBuild))
diff --git a/lang/python/python3/patches-setuptools/001-reproducible.patch b/lang/python/python3/patches-setuptools/001-reproducible.patch
new file mode 100644
index 000000000..32edc568d
--- /dev/null
+++ b/lang/python/python3/patches-setuptools/001-reproducible.patch
@@ -0,0 +1,16 @@
+https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=848136
+https://sources.debian.org/patches/python-setuptools/40.8.0-1/reproducible.diff/
+
+Index: b/setuptools/command/easy_install.py
+===================================================================
+--- a/setuptools/command/easy_install.py
++++ b/setuptools/command/easy_install.py
+@@ -436,7 +436,7 @@ consider to install to another location,
+             for spec in self.args:
+                 self.easy_install(spec, not self.no_deps)
+             if self.record:
+-                outputs = self.outputs
++                outputs = list(sorted(self.outputs))
+                 if self.root:  # strip any package prefix
+                     root_len = len(self.root)
+                     for counter in range(len(outputs)):
diff --git a/lang/python/python3/patches-setuptools/002-sorted-requires.patch b/lang/python/python3/patches-setuptools/002-sorted-requires.patch
new file mode 100644
index 000000000..2ad4795b3
--- /dev/null
+++ b/lang/python/python3/patches-setuptools/002-sorted-requires.patch
@@ -0,0 +1,16 @@
+https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=804249
+https://sources.debian.org/patches/python-setuptools/40.8.0-1/sorted-requires.diff/
+
+Index: b/setuptools/command/egg_info.py
+===================================================================
+--- a/setuptools/command/egg_info.py
++++ b/setuptools/command/egg_info.py
+@@ -621,7 +621,7 @@ def warn_depends_obsolete(cmd, basename,
+ def _write_requirements(stream, reqs):
+     lines = yield_lines(reqs or ())
+     append_cr = lambda line: line + '\n'
+-    lines = map(append_cr, lines)
++    lines = map(append_cr, sorted(lines))
+     stream.writelines(lines)
+ 
+ 
diff --git a/lang/python/python3/patches-setuptools/003-PKG-INFO-output-reproducible.patch b/lang/python/python3/patches-setuptools/003-PKG-INFO-output-reproducible.patch
new file mode 100644
index 000000000..15f34dcdb
--- /dev/null
+++ b/lang/python/python3/patches-setuptools/003-PKG-INFO-output-reproducible.patch
@@ -0,0 +1,14 @@
+https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=894215
+https://sources.debian.org/patches/python-setuptools/40.8.0-1/PKG-INFO-output-reproducible.diff/
+
+--- a/setuptools/dist.py
++++ b/setuptools/dist.py
+@@ -191,7 +191,7 @@ def write_pkg_file(self, file):
+             self.long_description_content_type
+         )
+     if self.provides_extras:
+-        for extra in self.provides_extras:
++        for extra in sorted(self.provides_extras):
+             write_field('Provides-Extra', extra)
+ 
+