Merge master into staging-next

This commit is contained in:
github-actions[bot]
2021-05-22 00:56:03 +00:00
committed by GitHub
87 changed files with 3511 additions and 1315 deletions
+26
View File
@@ -0,0 +1,26 @@
# GNOME Shell extensions
All extensions are packaged automatically. They can be found in the `pkgs.gnomeXYExtensions` for XY being a GNOME version. The package names are the extensions UUID, which can be a bit unwieldy to use. `pkgs.gnomeExtensions` is a set of manually curated extensions that match the current `gnome.gnome-shell` versions. Their name is human-friendly, compared to the other extensions sets. Some of its extensions are manually packaged.
## Automatically packaged extensions
The actual packages are created by `buildGnomeExtensions.nix`, provided the correct arguments are fed into it. The important extension data is stored in `extensions.json`, one line/item per extension. That file is generated by running `update-extensions.py`. Furthermore, the automatic generated names are dumped in `collisions.json` for manual inspection. `extensionRenames.nix` contains provides new names for all extensions that collide.
### Extensions updates
For everyday updates,
1. Run `update-extensions.py`.
2. Update `extensionRenames.nix` according to the comment at the top.
For GNOME updates,
1. Add a new `gnomeXYExtensions` set
2. Remove old ones for GNOME versions we dont want to support any more
3. Update `supported_versions` in `./update-extensions.py` and re-run it
4. Change `gnomeExtensions` to the new version
5. Update `./extensionsRenames.nix` accordingly
## Manually packaged extensions
Manually packaged extensions overwrite some of the automatically packaged ones in `pkgs.gnomeExtensions`. They are listed in `manuallyPackaged.nix`, every extension has its own sub-folder.
@@ -0,0 +1,54 @@
{ pkgs, lib, stdenv, fetchzip }:
let
buildGnomeExtension = {
# Every gnome extension has a UUID. It's the name of the extension folder once unpacked
# and can always be found in the metadata.json of every extension.
uuid,
name,
pname,
description,
# extensions.gnome.org extension URL
link,
# Extension version numbers are integers
version,
sha256,
# Hex-encoded string of JSON bytes
metadata,
}:
stdenv.mkDerivation {
inherit pname;
version = builtins.toString version;
src = fetchzip {
url = "https://extensions.gnome.org/extension-data/${
builtins.replaceStrings [ "@" ] [ "" ] uuid
}.v${builtins.toString version}.shell-extension.zip";
inherit sha256;
stripRoot = false;
# The download URL may change content over time. This is because the
# metadata.json is automatically generated, and parts of it can be changed
# without making a new release. We simply substitute the possibly changed fields
# with their content from when we last updated, and thus get a deterministic output
# hash.
extraPostFetch = ''
echo "${metadata}" | base64 --decode > $out/metadata.json
'';
};
buildCommand = ''
mkdir -p $out/share/gnome-shell/extensions/
cp -r -T $src $out/share/gnome-shell/extensions/${uuid}
'';
meta = {
description = builtins.head (lib.splitString "\n" description);
longDescription = description;
homepage = link;
license = lib.licenses.gpl2Plus; # https://wiki.gnome.org/Projects/GnomeShell/Extensions/Review#Licensing
maintainers = with lib.maintainers; [ piegames ];
};
# Store the extension's UUID, because we might need it at some places
passthru.extensionUuid = uuid;
};
in
lib.makeOverridable buildGnomeExtension
@@ -0,0 +1,42 @@
{
"38": {
"applications-menu": [
"apps-menu@gnome-shell-extensions.gcampax.github.com",
"Applications_Menu@rmy.pobox.com"
],
"workspace-indicator": [
"workspace-indicator@gnome-shell-extensions.gcampax.github.com",
"horizontal-workspace-indicator@tty2.io"
],
"lock-keys": [
"lockkeys@vaina.lt",
"lockkeys@fawtytoo"
],
"fuzzy-clock": [
"Fuzzy_Clock@dallagi",
"fuzzy-clock@keepawayfromfire.co.uk"
],
"transparent-window": [
"transparent-window@pbxqdown.github.com",
"transparentwindows.mdirshad07"
],
"floating-dock": [
"floatingDock@sun.wxg@gmail.com",
"floating-dock@nandoferreira_prof@hotmail.com"
]
},
"40": {
"applications-menu": [
"apps-menu@gnome-shell-extensions.gcampax.github.com",
"Applications_Menu@rmy.pobox.com"
],
"workspace-indicator": [
"workspace-indicator@gnome-shell-extensions.gcampax.github.com",
"horizontal-workspace-indicator@tty2.io"
],
"lock-keys": [
"lockkeys@vaina.lt",
"lockkeys@fawtytoo"
]
}
}
@@ -0,0 +1,73 @@
{ lib
, callPackage
, config
}:
let
buildShellExtension = callPackage ./buildGnomeExtension.nix { };
# Index of all scraped extensions (with supported versions)
extensionsIndex = lib.importJSON ./extensions.json;
# A list of UUIDs that have the same pname and we need to rename them
extensionRenames = import ./extensionRenames.nix;
# Take all extensions from the index that match the gnome version, build them and put them into a list of derivations
produceExtensionsList = shell-version:
lib.trivial.pipe extensionsIndex [
# Does a given extension match our current shell version?
(builtins.filter
(extension: (builtins.hasAttr shell-version extension."shell_version_map"))
)
# Take in an `extension` object from the JSON and transform it into the correct args to call `buildShellExtension`
(map
(extension: {
inherit (extension) uuid name description link pname;
inherit (extension.shell_version_map.${shell-version}) version sha256 metadata;
})
)
# Build them
(map buildShellExtension)
];
# Map the list of extensions to an attrset based on the UUID as key
mapUuidNames = extensions:
lib.trivial.pipe extensions [
(map (extension: lib.nameValuePair extension.extensionUuid extension))
builtins.listToAttrs
];
# Map the list of extensions to an attrset based on the pname as key, which is more human readable than the UUID
# We also take care of conflict renaming in here
mapReadableNames = extensionsList: lib.trivial.pipe extensionsList [
# Filter out all extensions that map to null
(lib.filter (extension:
!(
(builtins.hasAttr extension.extensionUuid extensionRenames)
&& ((builtins.getAttr extension.extensionUuid extensionRenames) == null)
)
))
# Map all extensions to their pname, with potential overwrites
(map (extension:
lib.nameValuePair (extensionRenames.${extension.extensionUuid} or extension.pname) extension
))
builtins.listToAttrs
];
in rec {
inherit buildShellExtension;
gnome38Extensions = mapUuidNames (produceExtensionsList "38");
gnome40Extensions = mapUuidNames (produceExtensionsList "40");
gnomeExtensions = lib.recurseIntoAttrs (
(mapReadableNames (produceExtensionsList "40"))
// (callPackage ./manuallyPackaged.nix {})
// lib.optionalAttrs (config.allowAliases or false) {
unite-shell = gnomeExtensions.unite; # added 2021-01-19
arc-menu = gnomeExtensions.arcmenu; # added 2021-02-14
nohotcorner = throw "gnomeExtensions.nohotcorner removed since 2019-10-09: Since 3.34, it is a part of GNOME Shell configurable through GNOME Tweaks.";
mediaplayer = throw "gnomeExtensions.mediaplayer deprecated since 2019-09-23: retired upstream https://github.com/JasonLG1979/gnome-shell-extensions-mediaplayer/blob/master/README.md";
}
);
}
@@ -0,0 +1,29 @@
# A list of UUIDs that have the same pname and we need to rename them
# MAINTENANCE:
# - Every item from ./collisions.json (for the respective Shell version) should have an entry in here
# - Set the value to `null` for filtering (duplicate or unmaintained extensions)
# - Sort the entries in order of appearance in the collisions.json
{
"apps-menu@gnome-shell-extensions.gcampax.github.com" = "applications-menu";
"Applications_Menu@rmy.pobox.com" = "frippery-applications-menu";
"workspace-indicator@gnome-shell-extensions.gcampax.github.com" = "workspace-indicator";
"horizontal-workspace-indicator@tty2.io" = "workspace-indicator-2";
"lockkeys@vaina.lt" = "lock-keys";
"lockkeys@fawtytoo" = "lock-keys-2";
# These are conflicts for 3.38 extensions. They will very probably come back
# once more of them support 40.
# See https://github.com/pbxqdown/gnome-shell-extension-transparent-window/issues/12#issuecomment-800765381
#"transparent-window@pbxqdown.github.com" = "transparent-window";
#"transparentwindows.mdirshad07" = null;
#"floatingDock@sun.wxg@gmail.com" = "floating-dock";
#"floating-dock@nandoferreira_prof@hotmail.com" = "floating-dock-2";
# That extension is broken because of https://github.com/NixOS/nixpkgs/issues/118612
#"flypie@schneegans.github.com" = null;
}
File diff suppressed because one or more lines are too long
@@ -0,0 +1,40 @@
{ callPackage }:
{
appindicator = callPackage ./appindicator { };
arcmenu = callPackage ./arcmenu { };
caffeine = callPackage ./caffeine { };
clipboard-indicator = callPackage ./clipboard-indicator { };
clock-override = callPackage ./clock-override { };
dash-to-dock = callPackage ./dash-to-dock { };
dash-to-panel = callPackage ./dash-to-panel { };
disable-unredirect = callPackage ./disable-unredirect { };
draw-on-your-screen = callPackage ./draw-on-your-screen { };
drop-down-terminal = callPackage ./drop-down-terminal { };
dynamic-panel-transparency = callPackage ./dynamic-panel-transparency { };
easyScreenCast = callPackage ./EasyScreenCast { };
emoji-selector = callPackage ./emoji-selector { };
freon = callPackage ./freon { };
fuzzy-app-search = callPackage ./fuzzy-app-search { };
gsconnect = callPackage ./gsconnect { };
hot-edge = callPackage ./hot-edge { };
icon-hider = callPackage ./icon-hider { };
impatience = callPackage ./impatience { };
material-shell = callPackage ./material-shell { };
mpris-indicator-button = callPackage ./mpris-indicator-button { };
night-theme-switcher = callPackage ./night-theme-switcher { };
no-title-bar = callPackage ./no-title-bar { };
noannoyance = callPackage ./noannoyance { };
paperwm = callPackage ./paperwm { };
pidgin-im-integration = callPackage ./pidgin-im-integration { };
remove-dropdown-arrows = callPackage ./remove-dropdown-arrows { };
sound-output-device-chooser = callPackage ./sound-output-device-chooser { };
system-monitor = callPackage ./system-monitor { };
taskwhisperer = callPackage ./taskwhisperer { };
tilingnome = callPackage ./tilingnome { };
timepp = callPackage ./timepp { };
topicons-plus = callPackage ./topicons-plus { };
unite = callPackage ./unite { };
window-corner-preview = callPackage ./window-corner-preview { };
window-is-ready-remover = callPackage ./window-is-ready-remover { };
workspace-matrix = callPackage ./workspace-matrix { };
}
+284
View File
@@ -0,0 +1,284 @@
#!/usr/bin/env nix-shell
#!nix-shell -I nixpkgs=../../../.. -i python3 -p python3
import json
import urllib.request
import urllib.error
from typing import List, Dict, Optional, Any, Tuple
import logging
from operator import itemgetter
import subprocess
import zipfile
import io
import base64
# We don't want all those deprecated legacy extensions
# Group extensions by GNOME "major" version for compatibility reasons
supported_versions = {
"38": "3.38",
"40": "40",
}
# Some type alias to increase readility of complex compound types
PackageName = str
ShellVersion = str
Uuid = str
ExtensionVersion = int
# Keep track of all names that have been used till now to detect collisions.
# This works because we deterministically process all extensions in historical order
# The outer dict level is the shell version, as we are tracking duplicates only per same Shell version.
# key: shell version, value: Dict with key: pname, value: list of UUIDs with that pname
package_name_registry: Dict[ShellVersion, Dict[PackageName, List[Uuid]]] = {}
for shell_version in supported_versions.keys():
package_name_registry[shell_version] = {}
def fetch_extension_data(uuid: str, version: str) -> Tuple[str, str]:
"""
Download the extension and hash it. We use `nix-prefetch-url` for this for efficiency reasons.
Returns a tuple with the hash (Nix-compatible) of the zip file's content and the base64-encoded content of its metadata.json.
"""
# The download URLs follow this schema
uuid = uuid.replace("@", "")
url: str = f"https://extensions.gnome.org/extension-data/{uuid}.v{version}.shell-extension.zip"
# Yes, we download that file three times:
# The first time is for the maintainter, so they may have a personal backup to fix potential issues
# subprocess.run(
# ["wget", url], capture_output=True, text=True
# )
# The second time, we extract the metadata.json because we need it too
with urllib.request.urlopen(url) as response:
data = zipfile.ZipFile(io.BytesIO(response.read()), 'r')
metadata = base64.b64encode(data.read('metadata.json')).decode()
# The third time is to get the file into the store and to get its hash
hash = subprocess.run(
["nix-prefetch-url", "--unpack", url], capture_output=True, text=True
).stdout.strip()
return hash, metadata
def generate_extension_versions(
extension_version_map: Dict[ShellVersion, ExtensionVersion], uuid: str
) -> Dict[ShellVersion, Dict[str, str]]:
"""
Takes in a mapping from shell versions to extension versions and transforms it the way we need it:
- Only take one extension version per GNOME Shell major version (as per `supported_versions`)
- Filter out versions that only support old GNOME versions
- Download the extension and hash it
"""
extension_versions: Dict[ShellVersion, Dict[str, str]] = {}
for shell_version, version_prefix in supported_versions.items():
# Newest compatible extension version
extension_version: Optional[int] = max(
(
int(ext_ver)
for shell_ver, ext_ver in extension_version_map.items()
if (shell_ver.startswith(version_prefix))
),
default=None,
)
# Extension is not compatible with this GNOME version
if not extension_version:
continue
logging.debug(
f"[{shell_version}] Downloading '{uuid}' v{extension_version}"
)
sha256, metadata = fetch_extension_data(uuid, str(extension_version))
extension_versions[shell_version] = {
"version": str(extension_version),
"sha256": sha256,
# The downloads are impure, their metadata.json may change at any time.
# Thus, be back it up / pin it to remain deterministic
# Upstream issue: https://gitlab.gnome.org/Infrastructure/extensions-web/-/issues/137
"metadata": metadata,
}
return extension_versions
def pname_from_url(url: str) -> Tuple[str, str]:
"""
Parse something like "/extension/1475/battery-time/" and output ("battery-time", "1475")
"""
url = url.split("/") # type: ignore
return (url[3], url[2])
def process_extension(extension: Dict[str, Any]) -> Optional[Dict[str, Any]]:
"""
Process an extension. It takes in raw scraped data and downloads all the necessary information that buildGnomeExtension.nix requires
Input: a json object of one extension queried from the site. It has the following schema (only important key listed):
{
"uuid": str,
"name": str,
"description": str,
"link": str,
"shell_version_map": {
str: { "version": int, … },
},
}
"uuid" is an extension UUID that looks like this (most of the time): "extension-name@username.domain.tld".
Don't make any assumptions on it, and treat it like an opaque string!
"link" follows the following schema: "/extension/$number/$string/"
The number is monotonically increasing and unique to every extension.
The string is usually derived from the extensions's name (but shortened, kebab-cased and URL friendly).
It may diverge from the actual name.
The keys of "shell_version_map" are GNOME Shell version numbers.
Output: a json object to be stored, or None if the extension should be skipped. Schema:
{
"uuid": str,
"name": str,
"pname": str,
"description": str,
"link": str,
"shell_version_map": {
str: { "version": int, "sha256": str, "metadata": <hex> },
}
}
Only "uuid" gets passed along unmodified. "name", "description" and "link" are taken from the input, but sanitized.
"pname" gets generated from other fields and "shell_version_map" has a completely different structure than the input
field with the same name.
"""
uuid = extension["uuid"]
# Yeah, there are some extensions without any releases
if not extension["shell_version_map"]:
return None
logging.info(f"Processing '{uuid}'")
# Input is a mapping str -> { version: int, … }
# We want to map shell versions to extension versions
shell_version_map: Dict[ShellVersion, int] = {
k: v["version"] for k, v in extension["shell_version_map"].items()
}
# Transform shell_version_map to be more useful for us. Also throw away unwanted versions
shell_version_map: Dict[ShellVersion, Dict[str, str]] = generate_extension_versions(shell_version_map, uuid) # type: ignore
# No compatible versions found
if not shell_version_map:
return None
# Fetch a human-readable name for the package.
(pname, _pname_id) = pname_from_url(extension["link"])
for shell_version in shell_version_map.keys():
if pname in package_name_registry[shell_version]:
logging.warning(f"Package name '{pname}' is colliding.")
package_name_registry[shell_version][pname].append(uuid)
else:
package_name_registry[shell_version][pname] = [uuid]
return {
"uuid": uuid,
"name": extension["name"],
"pname": pname,
"description": extension["description"],
"link": "https://extensions.gnome.org" + extension["link"],
"shell_version_map": shell_version_map,
}
def scrape_extensions_index() -> List[Dict[str, Any]]:
"""
Scrape the list of extensions by sending search queries to the API. We simply go over it
page by page until we hit a non-full page or a 404 error.
The returned list is sorted by the age of the extension, in order to be deterministic.
"""
page = 0
extensions = []
while True:
page += 1
logging.info("Scraping page " + str(page))
try:
with urllib.request.urlopen(
f"https://extensions.gnome.org/extension-query/?n_per_page=25&page={page}"
) as response:
data = json.loads(response.read().decode())["extensions"]
responseLength = len(data)
for extension in data:
extensions.append(extension)
# If our page isn't "full", it must have been the last one
if responseLength < 25:
logging.debug(
f"\tThis page only has {responseLength} entries, so it must be the last one."
)
break
except urllib.error.HTTPError as e:
if e.code == 404:
# We reached past the last page and are done now
break
else:
raise
# `pk` is the primary key in the extensions.gnome.org database. Sorting on it will give us a stable,
# deterministic ordering.
extensions.sort(key=itemgetter("pk"))
return extensions
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
raw_extensions = scrape_extensions_index()
logging.info(f"Downloaded {len(raw_extensions)} extensions. Processing …")
processed_extensions: List[Dict[str, Any]] = []
for num, raw_extension in enumerate(raw_extensions):
processed_extension = process_extension(raw_extension)
if processed_extension:
processed_extensions.append(processed_extension)
logging.debug(f"Processed {num + 1} / {len(raw_extensions)}")
logging.info(
f"Done. Writing results to extensions.json ({len(processed_extensions)} extensions in total)"
)
with open("extensions.json", "w") as out:
# Manually pretty-print the outer level, but then do one compact line per extension
# This allows for the diffs to be manageable (one line of change per extension) despite their quantity
for index, extension in enumerate(processed_extensions):
if index == 0:
out.write("[ ")
else:
out.write(", ")
json.dump(extension, out, ensure_ascii=False)
out.write("\n")
out.write("]\n")
with open("extensions.json", "r") as out:
# Check that the generated file actually is valid JSON, just to be sure
json.load(out)
logging.info(
"Done. Writing name collisions to collisions.json (please check manually)"
)
with open("collisions.json", "w") as out:
# Filter out those that are not duplicates
package_name_registry_filtered: Dict[ShellVersion, Dict[PackageName, List[Uuid]]] = {
# The outer level keys are shell versions
shell_version: {
# The inner keys are extension names, with a list of all extensions with that name as value.
pname: extensions for pname, extensions in collisions.items() if len(extensions) > 1
} for shell_version, collisions in package_name_registry.items()
}
json.dump(package_name_registry_filtered, out, indent=2, ensure_ascii=False)
out.write("\n")