diff --git a/pkgs/development/python-modules/fb-re2/default.nix b/pkgs/development/python-modules/fb-re2/default.nix new file mode 100644 index 00000000000..88a75d29fd1 --- /dev/null +++ b/pkgs/development/python-modules/fb-re2/default.nix @@ -0,0 +1,30 @@ +{ lib +, buildPythonPackage +, fetchPypi +, re2 +, pytest +}: + +buildPythonPackage rec { + pname = "fb-re2"; + version = "1.0.6"; + + src = fetchPypi { + inherit pname version; + sha256 = "0wd97qdcafcca90s6692g2dmzl5n6cdvm20vn7pmag3l9gvx395c"; + }; + + buildInputs = [ re2 ]; + + checkInputs = [ pytest ]; + checkPhase = '' + py.test + ''; + + meta = { + description = "Python wrapper for Google's RE2"; + homepage = https://github.com/facebook/pyre2; + license = lib.licenses.bsd3; + maintainers = with lib.maintainers; [ ivan ]; + }; +} diff --git a/pkgs/development/python-modules/ludios_wpull/default.nix b/pkgs/development/python-modules/ludios_wpull/default.nix new file mode 100644 index 00000000000..1c707b32031 --- /dev/null +++ b/pkgs/development/python-modules/ludios_wpull/default.nix @@ -0,0 +1,39 @@ +{ lib +, buildPythonPackage +, fetchFromGitHub +, isPy3k +, chardet +, dnspython +, html5-parser +, lxml +, namedlist +, sqlalchemy +, tornado_4 +, Yapsy +}: + +buildPythonPackage rec { + pname = "ludios_wpull"; + version = "3.0.7"; + + disabled = !isPy3k; + + src = fetchFromGitHub { + rev = "${version}"; + owner = "ludios"; + repo = "wpull"; + sha256 = "1j96avm0ynbazypzp766wh26n4qc73y7wgsiqfrdfl6x7rx20wgf"; + }; + + propagatedBuildInputs = [ chardet dnspython html5-parser lxml namedlist sqlalchemy tornado_4 Yapsy ]; + + # Test suite has tests that fail on all platforms + doCheck = false; + + meta = { + description = "Web crawler; fork of wpull used by grab-site"; + homepage = https://github.com/ludios/wpull; + license = lib.licenses.gpl3; + maintainers = with lib.maintainers; [ ivan ]; + }; +} diff --git a/pkgs/development/python-modules/namedlist/default.nix b/pkgs/development/python-modules/namedlist/default.nix new file mode 100644 index 00000000000..ae07076bb05 --- /dev/null +++ b/pkgs/development/python-modules/namedlist/default.nix @@ -0,0 +1,33 @@ +{ lib +, buildPythonPackage +, fetchPypi +, pytest +}: + +buildPythonPackage rec { + pname = "namedlist"; + version = "1.7"; + + src = fetchPypi { + inherit pname version; + sha256 = "11n9c4a5ak9971awkf1g92m6mcmiprhrw98ik2cmjsqxmz73j2qr"; + }; + + # Test file has a `unittest.main()` at the bottom that fails the tests; + # py.test can run the tests without it. + postPatch = '' + substituteInPlace test/test_namedlist.py --replace "unittest.main()" "" + ''; + + checkInputs = [ pytest ]; + checkPhase = '' + py.test + ''; + + meta = { + description = "Similar to namedtuple, but instances are mutable"; + homepage = https://bitbucket.org/ericvsmith/namedlist; + license = lib.licenses.asl20; + maintainers = with lib.maintainers; [ ivan ]; + }; +} diff --git a/pkgs/tools/backup/grab-site/default.nix b/pkgs/tools/backup/grab-site/default.nix new file mode 100644 index 00000000000..7209afd8e7e --- /dev/null +++ b/pkgs/tools/backup/grab-site/default.nix @@ -0,0 +1,30 @@ +{ stdenv, python3Packages, fetchFromGitHub }: + +python3Packages.buildPythonApplication rec { + version = "2.1.11"; + name = "grab-site-${version}"; + + src = fetchFromGitHub { + rev = "${version}"; + owner = "ludios"; + repo = "grab-site"; + sha256 = "0w24ngr2b7nipqiwkxpql2467b5aq2vbknkb0sry6a457kb5ppsl"; + }; + + propagatedBuildInputs = with python3Packages; [ + click ludios_wpull manhole lmdb autobahn fb-re2 websockets cchardet + ]; + + checkPhase = '' + export PATH=$PATH:$out/bin + bash ./tests/offline-tests + ''; + + meta = with stdenv.lib; { + description = "Crawler for web archiving with WARC output"; + homepage = https://github.com/ludios/grab-site; + license = licenses.mit; + maintainers = with maintainers; [ ivan ]; + platforms = platforms.all; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 209a90a1601..3e392af3f65 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -10004,6 +10004,8 @@ in gperftools = callPackage ../development/libraries/gperftools { }; + grab-site = callPackage ../tools/backup/grab-site { }; + grib-api = callPackage ../development/libraries/grib-api { }; grpc = callPackage ../development/libraries/grpc { }; diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 598e4839b9a..f4bb2c0ed0f 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -1671,6 +1671,8 @@ in { then callPackage ../development/python-modules/faulthandler {} else throw "faulthandler is built into ${python.executable}"; + fb-re2 = callPackage ../development/python-modules/fb-re2 { }; + flit = callPackage ../development/python-modules/flit { }; flowlogs_reader = callPackage ../development/python-modules/flowlogs_reader { }; @@ -1872,6 +1874,8 @@ in { lpod = callPackage ../development/python-modules/lpod { }; + ludios_wpull = callPackage ../development/python-modules/ludios_wpull { }; + luftdaten = callPackage ../development/python-modules/luftdaten { }; m2r = callPackage ../development/python-modules/m2r { }; @@ -3078,6 +3082,8 @@ in { namebench = callPackage ../development/python-modules/namebench { }; + namedlist = callPackage ../development/python-modules/namedlist { }; + nameparser = callPackage ../development/python-modules/nameparser { }; nbconvert = callPackage ../development/python-modules/nbconvert { };