From 0c999c7521ee52dd00566643f3600627783d2550 Mon Sep 17 00:00:00 2001 From: Chuck Date: Wed, 30 Oct 2019 17:32:15 -0700 Subject: [PATCH 1/2] itstool: 2.0.2 -> 2.0.6 To get python3 support. #63174 flipped itstool to python3, but itstool doesn't support python3 until 2.0.3 (and perhaps does not support it well until 2.0.5). Pressing forward instead of rolling back at worldofpeace's suggestion, who mentions that other distros seem to be able to ship recent versions of itstool. Tensions in this space seem two-fold. One set centers around libxml2 being a low-level C library with sharp edges, manual memory management, and performance concerns; the python libxml2 wrapper being quite thin (the most dubious character in this drama); and python's sentiment that it ought to be quite hard to crash the interpreter casually. This comes to a head in https://gitlab.gnome.org/GNOME/libxml2/issues/12 , where a use-after-free problem in idiomatic-looking python code is declared working-as-designed. The other set is around python3 being more UTF-8-aware than libxml2's python wrapper, such as https://bugzilla.gnome.org/show_bug.cgi?id=789714 and https://src.fedoraproject.org/rpms/libxml2/blob/master/f/libxml2-2.9.8-python3-unicode-errors.patch itstool is caught in this crossfire merely for being a widely-used python program that uses XML. --- pkgs/development/tools/misc/itstool/default.nix | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkgs/development/tools/misc/itstool/default.nix b/pkgs/development/tools/misc/itstool/default.nix index dd536685485..a5c3623b699 100644 --- a/pkgs/development/tools/misc/itstool/default.nix +++ b/pkgs/development/tools/misc/itstool/default.nix @@ -1,13 +1,11 @@ { stdenv, fetchurl, python3 }: stdenv.mkDerivation rec { - # 2.0.3+ breaks the build of gnome3.gnome-desktop - # https://github.com/itstool/itstool/issues/17 - name = "itstool-2.0.2"; + name = "itstool-2.0.6"; src = fetchurl { url = "http://files.itstool.org/itstool/${name}.tar.bz2"; - sha256 = "bf909fb59b11a646681a8534d5700fec99be83bb2c57badf8c1844512227033a"; + sha256 = "1acjgf8zlyk7qckdk19iqaca4jcmywd7vxjbcs1mm6kaf8icqcv2"; }; buildInputs = [ (python3.withPackages(ps: with ps; [ libxml2 ])) ]; From c0cecd0e6007382ff6c64d3d47a947895bc36ff6 Mon Sep 17 00:00:00 2001 From: Chuck Date: Thu, 31 Oct 2019 17:00:56 -0700 Subject: [PATCH 2/2] python3Packages.libxml2: Patch to work around python3 + utf-8 itstool crash 1. Gnumeric has unbalanced XML tags in its doc translations. 2. itstool's XML error handler tries to print this error with context. 3. libxml2's context snipper treats the data as bytes, not UTF-8. 4. python3Packages.libxml2 casts the context to a UTF-8 Python string. 5. itstool dereferences a null pointer. This patch intervenes at #4. In https://bugzilla.gnome.org/show_bug.cgi?id=789714#c4 , upstream suggests that intervening at #3 would be better -- that each of the four copies of xmlParserPrintFileContextInternal() have four additional UTF-8 problems, one of which is that the caret indicator ought to count "unicode characters" not bytes. But to position a caret correctly, a character count is not sufficient -- this would need to use icu's BiDi logic (with fallback to doing something wrong when libxml2 is configured not to use icu) -- which makes a 'correct' fix a much larger project than this simple band-aid. --- .../development/libraries/libxml2/default.nix | 14 +++++++++ .../libxml2/utf8-xmlErrorFuncHandler.patch | 30 +++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 pkgs/development/libraries/libxml2/utf8-xmlErrorFuncHandler.patch diff --git a/pkgs/development/libraries/libxml2/default.nix b/pkgs/development/libraries/libxml2/default.nix index 2d2fb89d595..7ed50f61224 100644 --- a/pkgs/development/libraries/libxml2/default.nix +++ b/pkgs/development/libraries/libxml2/default.nix @@ -14,6 +14,20 @@ stdenv.mkDerivation rec { url = "http://xmlsoft.org/sources/${pname}-${version}.tar.gz"; sha256 = "0wd881jzvqayx0ihzba29jl80k06xj9ywp16kxacdqs3064p1ywl"; }; + patches = [ + # Upstream bugs: + # https://bugzilla.gnome.org/show_bug.cgi?id=789714 + # https://gitlab.gnome.org/GNOME/libxml2/issues/64 + # Patch from https://bugzilla.opensuse.org/show_bug.cgi?id=1065270 , + # but only the UTF-8 part. + # Can also be mitigated by fixing malformed XML inputs, such as in + # https://gitlab.gnome.org/GNOME/gnumeric/merge_requests/3 . + # Other discussion: + # https://github.com/itstool/itstool/issues/22 + # https://github.com/NixOS/nixpkgs/pull/63174 + # https://github.com/NixOS/nixpkgs/pull/72342 + ./utf8-xmlErrorFuncHandler.patch + ]; outputs = [ "bin" "dev" "out" "man" "doc" ] ++ lib.optional pythonSupport "py" diff --git a/pkgs/development/libraries/libxml2/utf8-xmlErrorFuncHandler.patch b/pkgs/development/libraries/libxml2/utf8-xmlErrorFuncHandler.patch new file mode 100644 index 00000000000..9f4c99b0934 --- /dev/null +++ b/pkgs/development/libraries/libxml2/utf8-xmlErrorFuncHandler.patch @@ -0,0 +1,30 @@ +Index: libxml2-2.9.5/python/libxml.c +=================================================================== +--- libxml2-2.9.5.orig/python/libxml.c ++++ libxml2-2.9.5/python/libxml.c +@@ -1620,6 +1620,7 @@ libxml_xmlErrorFuncHandler(ATTRIBUTE_UNU + PyObject *message; + PyObject *result; + char str[1000]; ++ unsigned char *ptr = (unsigned char *)str; + + #ifdef DEBUG_ERROR + printf("libxml_xmlErrorFuncHandler(%p, %s, ...) called\n", ctx, msg); +@@ -1636,10 +1637,16 @@ libxml_xmlErrorFuncHandler(ATTRIBUTE_UNU + str[999] = 0; + va_end(ap); + ++#if PY_MAJOR_VERSION >= 3 ++ /* Ensure the error string doesn't start at UTF8 continuation. */ ++ while (*ptr && (*ptr & 0xc0) == 0x80) ++ ptr++; ++#endif ++ + list = PyTuple_New(2); + PyTuple_SetItem(list, 0, libxml_xmlPythonErrorFuncCtxt); + Py_XINCREF(libxml_xmlPythonErrorFuncCtxt); +- message = libxml_charPtrConstWrap(str); ++ message = libxml_charPtrConstWrap(ptr); + PyTuple_SetItem(list, 1, message); + result = PyEval_CallObject(libxml_xmlPythonErrorFuncHandler, list); + Py_XDECREF(list);